diff --git a/models/actions/artifact.go b/models/actions/artifact.go index 8a50c09179e..de95d72d3a4 100644 --- a/models/actions/artifact.go +++ b/models/actions/artifact.go @@ -9,10 +9,10 @@ package actions import ( "context" "errors" + "slices" "time" "gitea.dev/models/db" - "gitea.dev/modules/optional" "gitea.dev/modules/timeutil" "gitea.dev/modules/util" @@ -147,7 +147,7 @@ type FindArtifactsOptions struct { db.ListOptions RepoID int64 RunID int64 - RunAttemptID optional.Option[int64] // use optional to allow filtering by zero (legacy artifacts have run_attempt_id=0) + RunAttemptIDs []int64 // empty means every attempt; pass 0 to target legacy artifacts, which have run_attempt_id=0 ArtifactName string Status int FinalizedArtifactsV4 bool @@ -167,8 +167,8 @@ func (opts FindArtifactsOptions) ToConds() builder.Cond { if opts.RunID > 0 { cond = cond.And(builder.Eq{"run_id": opts.RunID}) } - if opts.RunAttemptID.Has() { - cond = cond.And(builder.Eq{"run_attempt_id": opts.RunAttemptID.Value()}) + if len(opts.RunAttemptIDs) > 0 { + cond = cond.And(builder.In("run_attempt_id", opts.RunAttemptIDs)) } if opts.ArtifactName != "" { cond = cond.And(builder.Eq{"artifact_name": opts.ArtifactName}) @@ -185,6 +185,27 @@ func (opts FindArtifactsOptions) ToConds() builder.Cond { return cond } +// FindReadableArtifacts returns the artifacts of opts.RunAttemptIDs, only keeps the ones from a newer attempt. +func FindReadableArtifacts(ctx context.Context, opts FindArtifactsOptions) ([]*ActionArtifact, error) { + arts, err := db.Find[ActionArtifact](ctx, opts) + if err != nil || len(opts.RunAttemptIDs) <= 1 { + return arts, err + } + return keepLatestAttemptArtifacts(arts), nil +} + +// keepLatestAttemptArtifacts keeps, per name, only the artifacts of the newest attempt that has it. +// A v3 artifact is one row per uploaded file, so the whole group of the winning attempt is kept. +func keepLatestAttemptArtifacts(arts []*ActionArtifact) []*ActionArtifact { + latest := make(map[string]int64) + for _, art := range arts { + latest[art.ArtifactName] = max(latest[art.ArtifactName], art.RunAttemptID) + } + return slices.DeleteFunc(arts, func(art *ActionArtifact) bool { + return art.RunAttemptID != latest[art.ArtifactName] + }) +} + // ActionArtifactMeta is the meta-data of an artifact type ActionArtifactMeta struct { ArtifactName string diff --git a/models/actions/artifact_test.go b/models/actions/artifact_test.go new file mode 100644 index 00000000000..f680483d53a --- /dev/null +++ b/models/actions/artifact_test.go @@ -0,0 +1,27 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package actions + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestKeepLatestAttemptArtifacts(t *testing.T) { + arts := []*ActionArtifact{ + {ID: 1, RunAttemptID: 1, ArtifactName: "inherited"}, + {ID: 2, RunAttemptID: 1, ArtifactName: "shadowed", ArtifactPath: "a.txt"}, + {ID: 3, RunAttemptID: 1, ArtifactName: "shadowed", ArtifactPath: "b.txt"}, + {ID: 4, RunAttemptID: 2, ArtifactName: "shadowed", ArtifactPath: "c.txt"}, + {ID: 5, RunAttemptID: 2, ArtifactName: "own"}, + } + + // the whole "shadowed" group of attempt 1 is dropped, its multi-file rows must not mix with attempt 2 + var ids []int64 + for _, art := range keepLatestAttemptArtifacts(arts) { + ids = append(ids, art.ID) + } + assert.Equal(t, []int64{1, 4, 5}, ids) +} diff --git a/models/actions/run_attempt.go b/models/actions/run_attempt.go index f686e448c4b..3e15c8c6a9b 100644 --- a/models/actions/run_attempt.go +++ b/models/actions/run_attempt.go @@ -11,6 +11,7 @@ import ( "gitea.dev/models/db" user_model "gitea.dev/models/user" + "gitea.dev/modules/container" "gitea.dev/modules/log" "gitea.dev/modules/timeutil" "gitea.dev/modules/util" @@ -96,6 +97,56 @@ func GetRunAttemptByRunIDAndAttemptNum(ctx context.Context, runID, attemptNum in return &attempt, nil } +// GetArtifactAttemptIDs returns the IDs of the attempts whose artifacts the job may read, newest first, +// always including the job's own attempt. +// An attempt that re-ran only some of the run's jobs keeps the artifacts of the attempt it re-ran from, +// because the jobs it passed through never upload them again; a rerun of the whole run starts over. +func GetArtifactAttemptIDs(ctx context.Context, job *ActionRunJob) ([]int64, error) { + if job.Attempt <= 1 || job.RunAttemptID == 0 { + return []int64{job.RunAttemptID}, nil + } + + attempts, err := ListRunAttemptsByRunID(ctx, job.RunID) + if err != nil { + return nil, err + } + // a newer attempt is never readable, and attempt 1 has nothing older to continue into + candidateIDs := container.FilterSlice(attempts, func(a *ActionRunAttempt) (int64, bool) { + return a.ID, a.Attempt > 1 && a.Attempt <= job.Attempt + }) + passThroughAttemptIDs, err := findPassThroughAttemptIDs(ctx, candidateIDs) + if err != nil { + return nil, err + } + + ids := make([]int64, 0, len(attempts)) + for _, attempt := range attempts { + if attempt.Attempt > job.Attempt { + continue + } + ids = append(ids, attempt.ID) + if !slices.Contains(passThroughAttemptIDs, attempt.ID) { + // stops at the first attempt that passed no job through + break + } + } + return ids, nil +} + +// findPassThroughAttemptIDs narrows the given attempts to those that were a rerun of selected jobs: +// only such a rerun clones jobs carrying a source task. +// TODO: best-effort. Needs a better way to distinguish between "partial re-run" and "full re-run". +func findPassThroughAttemptIDs(ctx context.Context, attemptIDs []int64) ([]int64, error) { + passThroughAttemptIDs := make([]int64, 0, len(attemptIDs)) + return passThroughAttemptIDs, db.GetEngine(ctx). + Table("action_run_job"). + Cols("run_attempt_id"). + In("run_attempt_id", attemptIDs). + Where("source_task_id <> 0"). + Distinct("run_attempt_id"). + Find(&passThroughAttemptIDs) +} + // FindConcurrentRunAttempts returns attempts in the given concurrency group and status set. // Results are unordered; callers must not depend on any particular row order. func FindConcurrentRunAttempts(ctx context.Context, repoID int64, concurrencyGroup string, statuses []Status) ([]*ActionRunAttempt, error) { diff --git a/routers/api/actions/artifacts.go b/routers/api/actions/artifacts.go index 5a577fdd3e4..ac6f6d1dc4e 100644 --- a/routers/api/actions/artifacts.go +++ b/routers/api/actions/artifacts.go @@ -66,6 +66,7 @@ import ( "errors" "fmt" "net/http" + "slices" "strconv" "strings" @@ -74,7 +75,6 @@ import ( "gitea.dev/modules/httplib" "gitea.dev/modules/json" "gitea.dev/modules/log" - "gitea.dev/modules/optional" "gitea.dev/modules/reqctx" "gitea.dev/modules/setting" "gitea.dev/modules/storage" @@ -337,15 +337,19 @@ type ( ) func (ar artifactRoutes) listArtifacts(ctx *ArtifactContext) { - _, runID, ok := validateRunID(ctx) + task, runID, ok := validateRunID(ctx) + if !ok { + return + } + attemptIDs, ok := readableArtifactAttemptIDs(ctx, task) if !ok { return } - artifacts, err := db.Find[actions.ActionArtifact](ctx, actions.FindArtifactsOptions{ - RunID: runID, - RunAttemptID: optional.Some(ctx.ActionTask.Job.RunAttemptID), - Status: int(actions.ArtifactStatusUploadConfirmed), + artifacts, err := actions.FindReadableArtifacts(ctx, actions.FindArtifactsOptions{ + RunID: runID, + RunAttemptIDs: attemptIDs, + Status: int(actions.ArtifactStatusUploadConfirmed), }) if err != nil { log.Error("Error getting artifacts: %v", err) @@ -398,7 +402,7 @@ type ( // getDownloadArtifactURL generates download url for each artifact func (ar artifactRoutes) getDownloadArtifactURL(ctx *ArtifactContext) { - _, runID, ok := validateRunID(ctx) + task, runID, ok := validateRunID(ctx) if !ok { return } @@ -408,11 +412,16 @@ func (ar artifactRoutes) getDownloadArtifactURL(ctx *ArtifactContext) { return } - artifacts, err := db.Find[actions.ActionArtifact](ctx, actions.FindArtifactsOptions{ - RunID: runID, - RunAttemptID: optional.Some(ctx.ActionTask.Job.RunAttemptID), - ArtifactName: itemPath, - Status: int(actions.ArtifactStatusUploadConfirmed), + attemptIDs, ok := readableArtifactAttemptIDs(ctx, task) + if !ok { + return + } + + artifacts, err := actions.FindReadableArtifacts(ctx, actions.FindArtifactsOptions{ + RunID: runID, + RunAttemptIDs: attemptIDs, + ArtifactName: itemPath, + Status: int(actions.ArtifactStatusUploadConfirmed), }) if err != nil { log.Error("Error getting artifacts: %v", err) @@ -462,7 +471,7 @@ func (ar artifactRoutes) getDownloadArtifactURL(ctx *ArtifactContext) { // downloadArtifact downloads artifact content func (ar artifactRoutes) downloadArtifact(ctx *ArtifactContext) { - _, runID, ok := validateRunID(ctx) + task, runID, ok := validateRunID(ctx) if !ok { return } @@ -484,10 +493,17 @@ func (ar artifactRoutes) downloadArtifact(ctx *ArtifactContext) { ctx.HTTPError(http.StatusBadRequest) return } - if ctx.ActionTask.Job.RunAttemptID > 0 && artifact.RunAttemptID != ctx.ActionTask.Job.RunAttemptID { - log.Error("Error mismatch runAttemptID and artifactID, task: %v, artifact: %v", ctx.ActionTask.Job.RunAttemptID, artifactID) - ctx.HTTPError(http.StatusBadRequest) - return + // resolving the readable attempts costs a query, and an artifact of the task's own attempt never needs it + if artifact.RunAttemptID != task.Job.RunAttemptID { + attemptIDs, ok := readableArtifactAttemptIDs(ctx, task) + if !ok { + return + } + if !slices.Contains(attemptIDs, artifact.RunAttemptID) { + log.Error("Error artifact %d belongs to run attempt %d, which the task cannot read: %v", artifactID, artifact.RunAttemptID, attemptIDs) + ctx.HTTPError(http.StatusBadRequest) + return + } } if artifact.Status != actions.ArtifactStatusUploadConfirmed { log.Error("Error artifact not found: %s", artifact.Status.ToString()) diff --git a/routers/api/actions/artifacts_chunks.go b/routers/api/actions/artifacts_chunks.go index 9c0ad8f02a9..1f9b3e86c28 100644 --- a/routers/api/actions/artifacts_chunks.go +++ b/routers/api/actions/artifacts_chunks.go @@ -20,7 +20,6 @@ import ( "gitea.dev/models/actions" "gitea.dev/models/db" "gitea.dev/modules/log" - "gitea.dev/modules/optional" "gitea.dev/modules/setting" "gitea.dev/modules/storage" ) @@ -261,9 +260,9 @@ func listOrderedChunksForArtifact(st storage.ObjectStorage, runID, artifactID in func mergeChunksForRun(ctx *ArtifactContext, st storage.ObjectStorage, runID, runAttemptID int64, artifactName string) error { // read all db artifacts by name artifacts, err := db.Find[actions.ActionArtifact](ctx, actions.FindArtifactsOptions{ - RunID: runID, - RunAttemptID: optional.Some(runAttemptID), - ArtifactName: artifactName, + RunID: runID, + RunAttemptIDs: []int64{runAttemptID}, + ArtifactName: artifactName, }) if err != nil { return err diff --git a/routers/api/actions/artifacts_utils.go b/routers/api/actions/artifacts_utils.go index 9494f8ab817..b4962939e02 100644 --- a/routers/api/actions/artifacts_utils.go +++ b/routers/api/actions/artifacts_utils.go @@ -43,7 +43,7 @@ func validateRunID(ctx *ArtifactContext) (*actions.ActionTask, int64, bool) { return task, runID, true } -func validateRunIDV4(ctx *ArtifactContext, rawRunID string) (*actions.ActionTask, int64, bool) { //nolint:unparam // ActionTask is never used +func validateRunIDV4(ctx *ArtifactContext, rawRunID string) (*actions.ActionTask, int64, bool) { task := ctx.ActionTask runID, err := strconv.ParseInt(rawRunID, 10, 64) if err != nil || task.Job.RunID != runID { @@ -54,6 +54,18 @@ func validateRunIDV4(ctx *ArtifactContext, rawRunID string) (*actions.ActionTask return task, runID, true } +// readableArtifactAttemptIDs resolves the attempts a task may read artifacts from: +// its own attempt, plus the attempts it inherits from when only a subset of the run's jobs was re-run. +func readableArtifactAttemptIDs(ctx *ArtifactContext, task *actions.ActionTask) ([]int64, bool) { + attemptIDs, err := actions.GetArtifactAttemptIDs(ctx, task.Job) + if err != nil { + log.Error("Error getting readable artifact attempts: %v", err) + ctx.HTTPError(http.StatusInternalServerError, "Error getting readable artifact attempts") + return nil, false + } + return attemptIDs, true +} + func validateArtifactHash(ctx *ArtifactContext, artifactName string) bool { paramHash := ctx.PathParam("artifact_hash") // use artifact name to create upload url diff --git a/routers/api/actions/artifactsv4.go b/routers/api/actions/artifactsv4.go index 12e7c17ae88..0fc95cfbec9 100644 --- a/routers/api/actions/artifactsv4.go +++ b/routers/api/actions/artifactsv4.go @@ -107,7 +107,6 @@ import ( actions_module "gitea.dev/modules/actions" "gitea.dev/modules/httplib" "gitea.dev/modules/log" - "gitea.dev/modules/optional" "gitea.dev/modules/setting" "gitea.dev/modules/storage" "gitea.dev/modules/util" @@ -262,9 +261,28 @@ func (r *artifactV4Routes) verifySignature(ctx *ArtifactContext, endp string) (* return task, artifactName, true } -func (r *artifactV4Routes) getArtifactByName(ctx *ArtifactContext, runID, runAttemptID int64, name string) (*actions_model.ActionArtifact, error) { +// getOwnAttemptArtifactByName resolves an artifact of the given attempt whatever its status, +// since upload and finalize work on the pending row they just created. +func (r *artifactV4Routes) getOwnAttemptArtifactByName(ctx *ArtifactContext, runID, runAttemptID int64, name string) (*actions_model.ActionArtifact, error) { + return r.findArtifactByName(ctx, runID, []int64{runAttemptID}, name, nil) +} + +// getDownloadableArtifactByName resolves the newest artifact with the given name within the attempts whose content can still be served, +// so a pending, deleted or expired row of a newer attempt does not shadow the confirmed copy inherited from an older one. +func (r *artifactV4Routes) getDownloadableArtifactByName(ctx *ArtifactContext, runID int64, runAttemptIDs []int64, name string) (*actions_model.ActionArtifact, error) { + return r.findArtifactByName(ctx, runID, runAttemptIDs, name, builder.Eq{"status": actions_model.ArtifactStatusUploadConfirmed}) +} + +func (r *artifactV4Routes) findArtifactByName(ctx *ArtifactContext, runID int64, runAttemptIDs []int64, name string, extraCond builder.Cond) (*actions_model.ActionArtifact, error) { + cond := builder.NewCond(). + And(builder.Eq{"run_id": runID, "artifact_name": name}, builder.Like{"content_encoding", "%/%"}). + And(builder.In("run_attempt_id", runAttemptIDs)) + if extraCond != nil { + cond = cond.And(extraCond) + } + var art actions_model.ActionArtifact - has, err := db.GetEngine(ctx).Where(builder.Eq{"run_id": runID, "run_attempt_id": runAttemptID, "artifact_name": name}, builder.Like{"content_encoding", "%/%"}).Get(&art) + has, err := db.GetEngine(ctx).Where(cond).OrderBy("run_attempt_id DESC, id DESC").Get(&art) if err != nil { return nil, err } else if !has { @@ -384,7 +402,7 @@ func (r *artifactV4Routes) uploadArtifact(ctx *ArtifactContext) { switch comp { case "block", "appendBlock": // get artifact by name - artifact, err := r.getArtifactByName(ctx, task.Job.RunID, task.Job.RunAttemptID, artifactName) + artifact, err := r.getOwnAttemptArtifactByName(ctx, task.Job.RunID, task.Job.RunAttemptID, artifactName) if err != nil { log.Error("Error artifact not found: %v", err) ctx.HTTPError(http.StatusNotFound, "Error artifact not found") @@ -471,7 +489,7 @@ func (r *artifactV4Routes) finalizeArtifact(ctx *ArtifactContext) { } // get artifact by name - artifact, err := r.getArtifactByName(ctx, runID, ctx.ActionTask.Job.RunAttemptID, req.Name) + artifact, err := r.getOwnAttemptArtifactByName(ctx, runID, ctx.ActionTask.Job.RunAttemptID, req.Name) if err != nil { log.Error("Error artifact not found: %v", err) ctx.HTTPError(http.StatusNotFound, "Error artifact not found") @@ -578,14 +596,18 @@ func (r *artifactV4Routes) listArtifacts(ctx *ArtifactContext) { if ok := r.parseProtobufBody(ctx, &req); !ok { return } - _, runID, ok := validateRunIDV4(ctx, req.WorkflowRunBackendId) + task, runID, ok := validateRunIDV4(ctx, req.WorkflowRunBackendId) + if !ok { + return + } + attemptIDs, ok := readableArtifactAttemptIDs(ctx, task) if !ok { return } - artifacts, err := db.Find[actions_model.ActionArtifact](ctx, actions_model.FindArtifactsOptions{ + artifacts, err := actions_model.FindReadableArtifacts(ctx, actions_model.FindArtifactsOptions{ RunID: runID, - RunAttemptID: optional.Some(ctx.ActionTask.Job.RunAttemptID), + RunAttemptIDs: attemptIDs, Status: int(actions_model.ArtifactStatusUploadConfirmed), FinalizedArtifactsV4: true, }) @@ -597,6 +619,8 @@ func (r *artifactV4Routes) listArtifacts(ctx *ArtifactContext) { list := []*ListArtifactsResponse_MonolithArtifact{} + // both filters pick from what this attempt may read, so they run after the shadowed artifacts are gone: + // a shadowed artifact is not downloadable either, GetSignedArtifactURL resolves by name table := map[string]*ListArtifactsResponse_MonolithArtifact{} for _, artifact := range artifacts { if _, ok := table[artifact.ArtifactName]; ok || req.IdFilter != nil && artifact.ID != req.IdFilter.Value || req.NameFilter != nil && artifact.ArtifactName != req.NameFilter.Value { @@ -631,7 +655,11 @@ func (r *artifactV4Routes) getSignedArtifactURL(ctx *ArtifactContext) { if ok := r.parseProtobufBody(ctx, &req); !ok { return } - _, runID, ok := validateRunIDV4(ctx, req.WorkflowRunBackendId) + task, runID, ok := validateRunIDV4(ctx, req.WorkflowRunBackendId) + if !ok { + return + } + attemptIDs, ok := readableArtifactAttemptIDs(ctx, task) if !ok { return } @@ -639,17 +667,12 @@ func (r *artifactV4Routes) getSignedArtifactURL(ctx *ArtifactContext) { artifactName := req.Name // get artifact by name - artifact, err := r.getArtifactByName(ctx, runID, ctx.ActionTask.Job.RunAttemptID, artifactName) + artifact, err := r.getDownloadableArtifactByName(ctx, runID, attemptIDs, artifactName) if err != nil { log.Error("Error artifact not found: %v", err) ctx.HTTPError(http.StatusNotFound, "Error artifact not found") return } - if artifact.Status != actions_model.ArtifactStatusUploadConfirmed { - log.Error("Error artifact not found: %s", artifact.Status.ToString()) - ctx.HTTPError(http.StatusNotFound, "Error artifact not found") - return - } respData := GetSignedArtifactURLResponse{} @@ -671,16 +694,15 @@ func (r *artifactV4Routes) downloadArtifact(ctx *ArtifactContext) { if !ok { return } - - // get artifact by name - artifact, err := r.getArtifactByName(ctx, task.Job.RunID, task.Job.RunAttemptID, artifactName) - if err != nil { - log.Error("Error artifact not found: %v", err) - ctx.HTTPError(http.StatusNotFound, "Error artifact not found") + attemptIDs, ok := readableArtifactAttemptIDs(ctx, task) + if !ok { return } - if artifact.Status != actions_model.ArtifactStatusUploadConfirmed { - log.Error("Error artifact not found: %s", artifact.Status.ToString()) + + // get artifact by name + artifact, err := r.getDownloadableArtifactByName(ctx, task.Job.RunID, attemptIDs, artifactName) + if err != nil { + log.Error("Error artifact not found: %v", err) ctx.HTTPError(http.StatusNotFound, "Error artifact not found") return } @@ -704,7 +726,7 @@ func (r *artifactV4Routes) deleteArtifact(ctx *ArtifactContext) { } // get artifact by name - artifact, err := r.getArtifactByName(ctx, runID, ctx.ActionTask.Job.RunAttemptID, req.Name) + artifact, err := r.getOwnAttemptArtifactByName(ctx, runID, ctx.ActionTask.Job.RunAttemptID, req.Name) if err != nil { log.Error("Error artifact not found: %v", err) ctx.HTTPError(http.StatusNotFound, "Error artifact not found") diff --git a/tests/integration/api_actions_artifact_test.go b/tests/integration/api_actions_artifact_test.go index af809db7569..ece8afda4c8 100644 --- a/tests/integration/api_actions_artifact_test.go +++ b/tests/integration/api_actions_artifact_test.go @@ -583,6 +583,10 @@ jobs: t.Run("testActionRunAttemptArtifactV4", func(t *testing.T) { testActionRunAttemptArtifactV4(t, repo, session, runner) }) + + t.Run("testPartialRerunArtifactInheritance", func(t *testing.T) { + testPartialRerunArtifactInheritance(t, user2, token, repo, session, runner) + }) }) } @@ -627,6 +631,107 @@ func testActionRunAttemptArtifactV3(t *testing.T, repo *repo_model.Repository, s assert.Equal(t, strings.Repeat("D", 32), sharedContent2) } +func testPartialRerunArtifactInheritance(t *testing.T, user *user_model.User, token string, repo *repo_model.Repository, session *TestSession, runner *mockRunner) { + wfTreePath := ".gitea/workflows/partial-rerun-artifact.yml" + wfFileContent := `name: partial-rerun-artifact +on: + workflow_dispatch: +jobs: + job1: + runs-on: ubuntu-latest + steps: + - run: echo 'job1' + job2: + runs-on: ubuntu-latest + needs: job1 + steps: + - run: echo 'job2' +` + opts := getWorkflowCreateFileOptions(user, repo.DefaultBranch, "create "+wfTreePath, wfFileContent) + createWorkflowFile(t, token, user.Name, repo.Name, wfTreePath, opts) + + req := NewRequestWithValues(t, "POST", fmt.Sprintf("/%s/%s/actions/run?workflow=%s", repo.OwnerName, repo.Name, "partial-rerun-artifact.yml"), map[string]string{ + "ref": "refs/heads/main", + }) + session.MakeRequest(t, req, http.StatusSeeOther) + + // job1 uploads the artifacts and succeeds + task1 := runner.fetchTask(t) + _, job1, run := getTaskAndJobAndRunByTaskID(t, task1.Id) + taskToken1 := task1.Context.GetFields()["gitea_runtime_token"].GetStringValue() + uploadTestArtifactFileV4(t, run.ID, job1.ID, taskToken1, "job1-only", strings.Repeat("A", 32)) + uploadTestArtifactFileV4(t, run.ID, job1.ID, taskToken1, "job1-shared", strings.Repeat("B", 32)) + // a v3 artifact is one row per file, so this one spans two rows + uploadTestArtifactFile(t, run.ID, taskToken1, "job1-v3", "a.txt", strings.Repeat("D", 32)) + uploadTestArtifactFile(t, run.ID, taskToken1, "job1-v3", "b.txt", strings.Repeat("E", 32)) + runner.execTask(t, task1, &mockTaskOutcome{result: runnerv1.Result_RESULT_SUCCESS}) + + // job2 fails + task2 := runner.fetchTask(t) + _, job2, _ := getTaskAndJobAndRunByTaskID(t, task2.Id) + runner.execTask(t, task2, &mockTaskOutcome{result: runnerv1.Result_RESULT_FAILURE}) + + // re-run only the failed job, so job1 is passed through and never uploads its artifacts again + req = NewRequest(t, "POST", fmt.Sprintf("/%s/%s/actions/runs/%d/rerun-failed", repo.OwnerName, repo.Name, run.ID)) + session.MakeRequest(t, req, http.StatusOK) + + task3 := runner.fetchTask(t) + _, job3, _ := getTaskAndJobAndRunByTaskID(t, task3.Id) + require.Equal(t, job2.JobID, job3.JobID) + require.NotEqual(t, job2.RunAttemptID, job3.RunAttemptID) + taskToken3 := task3.Context.GetFields()["gitea_runtime_token"].GetStringValue() + + // the new attempt inherits what the previous attempt uploaded + assert.ElementsMatch(t, []string{"job1-only", "job1-shared"}, listArtifactNamesForRunV4(t, run.ID, job3.ID, taskToken3)) + assert.Equal(t, strings.Repeat("A", 32), downloadArtifactContentV4ByTask(t, run.ID, job3.ID, taskToken3, "job1-only")) + assert.Contains(t, listArtifactNamesForRun(t, run.ID, taskToken3), "job1-v3") + + // a pending upload of this attempt must not shadow the confirmed copy it inherited + createTestArtifactV4(t, run.ID, job3.ID, taskToken3, "job1-only") + assert.Equal(t, strings.Repeat("A", 32), downloadArtifactContentV4ByTask(t, run.ID, job3.ID, taskToken3, "job1-only")) + + // both rows of the inherited v3 artifact are readable + inheritedV3 := getArtifactDownloadItemsForRun(t, run.ID, taskToken3, "job1-v3") + require.Len(t, inheritedV3, 2) + assert.Equal(t, strings.Repeat("D", 32), downloadArtifactItemContent(t, taskToken3, inheritedV3[0])) + + // uploading an inherited name in this attempt shadows the inherited artifact + inheritedSharedID := listArtifactIDForRunV4(t, run.ID, job3.ID, taskToken3, "job1-shared") + require.Len(t, listArtifactsByIDV4(t, run.ID, job3.ID, inheritedSharedID, taskToken3), 1) + uploadTestArtifactFileV4(t, run.ID, job3.ID, taskToken3, "job1-shared", strings.Repeat("C", 32)) + assert.ElementsMatch(t, []string{"job1-only", "job1-shared"}, listArtifactNamesForRunV4(t, run.ID, job3.ID, taskToken3)) + assert.Equal(t, strings.Repeat("C", 32), downloadArtifactContentV4ByTask(t, run.ID, job3.ID, taskToken3, "job1-shared")) + + // a shadowed artifact is not listed by id either: a download resolves by name + assert.Empty(t, listArtifactsByIDV4(t, run.ID, job3.ID, inheritedSharedID, taskToken3)) + + // the shadowed v3 artifact is dropped as a whole, its b.txt row must not survive next to the new a.txt + uploadTestArtifactFile(t, run.ID, taskToken3, "job1-v3", "a.txt", strings.Repeat("F", 32)) + shadowedV3 := getArtifactDownloadItemsForRun(t, run.ID, taskToken3, "job1-v3") + require.Len(t, shadowedV3, 1) + assert.Equal(t, strings.Repeat("F", 32), downloadArtifactItemContent(t, taskToken3, shadowedV3[0])) + + runner.execTask(t, task3, &mockTaskOutcome{result: runnerv1.Result_RESULT_SUCCESS}) +} + +func getArtifactDownloadItemsForRun(t *testing.T, runID int64, taskToken, artifactName string) []downloadArtifactResponseItem { + t.Helper() + + req := NewRequest(t, "GET", fmt.Sprintf("/api/actions_pipeline/_apis/pipelines/workflows/%d/artifacts/%x/download_url?itemPath=%s", runID, md5.Sum([]byte(artifactName)), artifactName)). + AddTokenAuth(taskToken) + resp := MakeRequest(t, req, http.StatusOK) + return DecodeJSON(t, resp, &downloadArtifactResponse{}).Value +} + +func downloadArtifactItemContent(t *testing.T, taskToken string, item downloadArtifactResponseItem) string { + t.Helper() + + idx := strings.Index(item.ContentLocation, "/api/actions_pipeline/_apis/pipelines/") + require.NotEqual(t, -1, idx) + req := NewRequest(t, "GET", item.ContentLocation[idx:]).AddTokenAuth(taskToken) + return MakeRequest(t, req, http.StatusOK).Body.String() +} + func uploadTestArtifactFile(t *testing.T, runID int64, authToken, artifactName, fileName, content string) { t.Helper() diff --git a/tests/integration/api_actions_artifact_v4_test.go b/tests/integration/api_actions_artifact_v4_test.go index fc9c0bc7517..16f8f6f4dad 100644 --- a/tests/integration/api_actions_artifact_v4_test.go +++ b/tests/integration/api_actions_artifact_v4_test.go @@ -947,7 +947,24 @@ func testActionRunAttemptArtifactV4(t *testing.T, repo *repo_model.Repository, s assert.Equal(t, strings.Repeat("D", 32), downloadRepoArtifactV4Content(t, session, sharedArtifactsResp.Entries[1].ArchiveDownloadURL)) } -func uploadTestArtifactFileV4(t *testing.T, runID, jobID int64, authToken, artifactName, content string) { +func downloadArtifactContentV4ByTask(t *testing.T, runID, jobID int64, taskToken, artifactName string) string { + t.Helper() + + req := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/GetSignedArtifactURL", toProtoJSON(&actions.GetSignedArtifactURLRequest{ + Name: artifactName, + WorkflowRunBackendId: strconv.FormatInt(runID, 10), + WorkflowJobRunBackendId: strconv.FormatInt(jobID, 10), + })).AddTokenAuth(taskToken) + resp := MakeRequest(t, req, http.StatusOK) + var urlResp actions.GetSignedArtifactURLResponse + require.NoError(t, protojson.Unmarshal(resp.Body.Bytes(), &urlResp)) + require.NotEmpty(t, urlResp.SignedUrl) + + return MakeRequest(t, NewRequest(t, "GET", urlResp.SignedUrl), http.StatusOK).Body.String() +} + +// createTestArtifactV4 only creates the artifact record, leaving it pending until it is uploaded and finalized +func createTestArtifactV4(t *testing.T, runID, jobID int64, authToken, artifactName string) string { t.Helper() req := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/CreateArtifact", toProtoJSON(&actions.CreateArtifactRequest{ @@ -958,11 +975,17 @@ func uploadTestArtifactFileV4(t *testing.T, runID, jobID int64, authToken, artif MimeType: wrapperspb.String("application/zip"), })).AddTokenAuth(authToken) resp := MakeRequest(t, req, http.StatusOK) - var uploadResp actions.CreateArtifactResponse - require.NoError(t, protojson.Unmarshal(resp.Body.Bytes(), &uploadResp)) - require.True(t, uploadResp.Ok) + var createResp actions.CreateArtifactResponse + require.NoError(t, protojson.Unmarshal(resp.Body.Bytes(), &createResp)) + require.True(t, createResp.Ok) + return createResp.SignedUploadUrl +} - req = NewRequestWithBody(t, "PUT", uploadResp.SignedUploadUrl+"&comp=appendBlock", strings.NewReader(content)) +func uploadTestArtifactFileV4(t *testing.T, runID, jobID int64, authToken, artifactName, content string) { + t.Helper() + + signedUploadURL := createTestArtifactV4(t, runID, jobID, authToken, artifactName) + req := NewRequestWithBody(t, "PUT", signedUploadURL+"&comp=appendBlock", strings.NewReader(content)) MakeRequest(t, req, http.StatusCreated) sum := sha256.Sum256([]byte(content)) @@ -973,30 +996,59 @@ func uploadTestArtifactFileV4(t *testing.T, runID, jobID int64, authToken, artif WorkflowRunBackendId: strconv.FormatInt(runID, 10), WorkflowJobRunBackendId: strconv.FormatInt(jobID, 10), })).AddTokenAuth(authToken) - resp = MakeRequest(t, req, http.StatusOK) + resp := MakeRequest(t, req, http.StatusOK) var finalizeResp actions.FinalizeArtifactResponse require.NoError(t, protojson.Unmarshal(resp.Body.Bytes(), &finalizeResp)) require.True(t, finalizeResp.Ok) } +func listArtifactsForRunV4(t *testing.T, taskToken string, req *actions.ListArtifactsRequest) []*actions.ListArtifactsResponse_MonolithArtifact { + t.Helper() + + httpReq := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/ListArtifacts", toProtoJSON(req)).AddTokenAuth(taskToken) + resp := MakeRequest(t, httpReq, http.StatusOK) + var listResp actions.ListArtifactsResponse + require.NoError(t, protojson.Unmarshal(resp.Body.Bytes(), &listResp)) + return listResp.Artifacts +} + func listArtifactNamesForRunV4(t *testing.T, runID, jobID int64, taskToken string) []string { t.Helper() - req := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/ListArtifacts", toProtoJSON(&actions.ListArtifactsRequest{ + artifacts := listArtifactsForRunV4(t, taskToken, &actions.ListArtifactsRequest{ WorkflowRunBackendId: strconv.FormatInt(runID, 10), WorkflowJobRunBackendId: strconv.FormatInt(jobID, 10), - })).AddTokenAuth(taskToken) - resp := MakeRequest(t, req, http.StatusOK) - var listResp actions.ListArtifactsResponse - require.NoError(t, protojson.Unmarshal(resp.Body.Bytes(), &listResp)) + }) - names := make([]string, 0, len(listResp.Artifacts)) - for _, item := range listResp.Artifacts { + names := make([]string, 0, len(artifacts)) + for _, item := range artifacts { names = append(names, item.Name) } return names } +func listArtifactIDForRunV4(t *testing.T, runID, jobID int64, taskToken, artifactName string) int64 { + t.Helper() + + artifacts := listArtifactsForRunV4(t, taskToken, &actions.ListArtifactsRequest{ + NameFilter: wrapperspb.String(artifactName), + WorkflowRunBackendId: strconv.FormatInt(runID, 10), + WorkflowJobRunBackendId: strconv.FormatInt(jobID, 10), + }) + require.Len(t, artifacts, 1) + return artifacts[0].DatabaseId +} + +func listArtifactsByIDV4(t *testing.T, runID, jobID, artifactID int64, taskToken string) []*actions.ListArtifactsResponse_MonolithArtifact { + t.Helper() + + return listArtifactsForRunV4(t, taskToken, &actions.ListArtifactsRequest{ + IdFilter: wrapperspb.Int64(artifactID), + WorkflowRunBackendId: strconv.FormatInt(runID, 10), + WorkflowJobRunBackendId: strconv.FormatInt(jobID, 10), + }) +} + func downloadRepoArtifactV4Content(t *testing.T, session *TestSession, archiveDownloadURL string) string { t.Helper()