fix(actions): let a rerun of selected jobs read the previous attempt's artifacts (#38857)

Fixes #38773

## Background

Artifacts became attempt-scoped in #37119, and the runner-facing
artifact APIs filter strictly by the attempt of the running job. "Re-run
failed jobs" creates a new attempt whose passed-through jobs never
upload their artifacts again, so a re-run job that downloads one of them
fails with "artifact not found".

## Fix

The read paths (v3 and v4 list and download) now resolve artifacts
across the running job's attempt plus the attempts it inherits from, and
an inherited artifact is shadowed by a same-named one from a newer
attempt.

## Note

GitHub's documentation does not document these behaviors. The
conclusions below are based on manual testing, so consistency with
GitHub cannot be guaranteed.

- In a "partial re-run", a job can download artifacts uploaded by an
earlier attempt, every attempt keeps its own copy of a name, and a
lookup by name resolves to the newest one.
- A full "Re-run all jobs" never downloads artifacts from earlier
attempts.

---------

Co-authored-by: bircni <bircni@icloud.com>

routers/api/actions/artifacts.go

@@ -66,6 +66,7 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"slices"
 	"strconv"
 	"strings"
 
@@ -74,7 +75,6 @@ import (
 	"gitea.dev/modules/httplib"
 	"gitea.dev/modules/json"
 	"gitea.dev/modules/log"
-	"gitea.dev/modules/optional"
 	"gitea.dev/modules/reqctx"
 	"gitea.dev/modules/setting"
 	"gitea.dev/modules/storage"
@@ -337,15 +337,19 @@ type (
 )
 
 func (ar artifactRoutes) listArtifacts(ctx *ArtifactContext) {
-	_, runID, ok := validateRunID(ctx)
+	task, runID, ok := validateRunID(ctx)
+	if !ok {
+		return
+	}
+	attemptIDs, ok := readableArtifactAttemptIDs(ctx, task)
 	if !ok {
 		return
 	}
 
-	artifacts, err := db.Find[actions.ActionArtifact](ctx, actions.FindArtifactsOptions{
-		RunID:        runID,
-		RunAttemptID: optional.Some(ctx.ActionTask.Job.RunAttemptID),
-		Status:       int(actions.ArtifactStatusUploadConfirmed),
+	artifacts, err := actions.FindReadableArtifacts(ctx, actions.FindArtifactsOptions{
+		RunID:         runID,
+		RunAttemptIDs: attemptIDs,
+		Status:        int(actions.ArtifactStatusUploadConfirmed),
 	})
 	if err != nil {
 		log.Error("Error getting artifacts: %v", err)
@@ -398,7 +402,7 @@ type (
 
 // getDownloadArtifactURL generates download url for each artifact
 func (ar artifactRoutes) getDownloadArtifactURL(ctx *ArtifactContext) {
-	_, runID, ok := validateRunID(ctx)
+	task, runID, ok := validateRunID(ctx)
 	if !ok {
 		return
 	}
@@ -408,11 +412,16 @@ func (ar artifactRoutes) getDownloadArtifactURL(ctx *ArtifactContext) {
 		return
 	}
 
-	artifacts, err := db.Find[actions.ActionArtifact](ctx, actions.FindArtifactsOptions{
-		RunID:        runID,
-		RunAttemptID: optional.Some(ctx.ActionTask.Job.RunAttemptID),
-		ArtifactName: itemPath,
-		Status:       int(actions.ArtifactStatusUploadConfirmed),
+	attemptIDs, ok := readableArtifactAttemptIDs(ctx, task)
+	if !ok {
+		return
+	}
+
+	artifacts, err := actions.FindReadableArtifacts(ctx, actions.FindArtifactsOptions{
+		RunID:         runID,
+		RunAttemptIDs: attemptIDs,
+		ArtifactName:  itemPath,
+		Status:        int(actions.ArtifactStatusUploadConfirmed),
 	})
 	if err != nil {
 		log.Error("Error getting artifacts: %v", err)
@@ -462,7 +471,7 @@ func (ar artifactRoutes) getDownloadArtifactURL(ctx *ArtifactContext) {
 
 // downloadArtifact downloads artifact content
 func (ar artifactRoutes) downloadArtifact(ctx *ArtifactContext) {
-	_, runID, ok := validateRunID(ctx)
+	task, runID, ok := validateRunID(ctx)
 	if !ok {
 		return
 	}
@@ -484,10 +493,17 @@ func (ar artifactRoutes) downloadArtifact(ctx *ArtifactContext) {
 		ctx.HTTPError(http.StatusBadRequest)
 		return
 	}
-	if ctx.ActionTask.Job.RunAttemptID > 0 && artifact.RunAttemptID != ctx.ActionTask.Job.RunAttemptID {
-		log.Error("Error mismatch runAttemptID and artifactID, task: %v, artifact: %v", ctx.ActionTask.Job.RunAttemptID, artifactID)
-		ctx.HTTPError(http.StatusBadRequest)
-		return
+	// resolving the readable attempts costs a query, and an artifact of the task's own attempt never needs it
+	if artifact.RunAttemptID != task.Job.RunAttemptID {
+		attemptIDs, ok := readableArtifactAttemptIDs(ctx, task)
+		if !ok {
+			return
+		}
+		if !slices.Contains(attemptIDs, artifact.RunAttemptID) {
+			log.Error("Error artifact %d belongs to run attempt %d, which the task cannot read: %v", artifactID, artifact.RunAttemptID, attemptIDs)
+			ctx.HTTPError(http.StatusBadRequest)
+			return
+		}
 	}
 	if artifact.Status != actions.ArtifactStatusUploadConfirmed {
 		log.Error("Error artifact not found: %s", artifact.Status.ToString())

routers/api/actions/artifacts_chunks.go

@@ -20,7 +20,6 @@ import (
 	"gitea.dev/models/actions"
 	"gitea.dev/models/db"
 	"gitea.dev/modules/log"
-	"gitea.dev/modules/optional"
 	"gitea.dev/modules/setting"
 	"gitea.dev/modules/storage"
 )
@@ -261,9 +260,9 @@ func listOrderedChunksForArtifact(st storage.ObjectStorage, runID, artifactID in
 func mergeChunksForRun(ctx *ArtifactContext, st storage.ObjectStorage, runID, runAttemptID int64, artifactName string) error {
 	// read all db artifacts by name
 	artifacts, err := db.Find[actions.ActionArtifact](ctx, actions.FindArtifactsOptions{
-		RunID:        runID,
-		RunAttemptID: optional.Some(runAttemptID),
-		ArtifactName: artifactName,
+		RunID:         runID,
+		RunAttemptIDs: []int64{runAttemptID},
+		ArtifactName:  artifactName,
 	})
 	if err != nil {
 		return err

routers/api/actions/artifacts_utils.go

@@ -43,7 +43,7 @@ func validateRunID(ctx *ArtifactContext) (*actions.ActionTask, int64, bool) {
 	return task, runID, true
 }
 
-func validateRunIDV4(ctx *ArtifactContext, rawRunID string) (*actions.ActionTask, int64, bool) { //nolint:unparam // ActionTask is never used
+func validateRunIDV4(ctx *ArtifactContext, rawRunID string) (*actions.ActionTask, int64, bool) {
 	task := ctx.ActionTask
 	runID, err := strconv.ParseInt(rawRunID, 10, 64)
 	if err != nil || task.Job.RunID != runID {
@@ -54,6 +54,18 @@ func validateRunIDV4(ctx *ArtifactContext, rawRunID string) (*actions.ActionTask
 	return task, runID, true
 }
 
+// readableArtifactAttemptIDs resolves the attempts a task may read artifacts from:
+// its own attempt, plus the attempts it inherits from when only a subset of the run's jobs was re-run.
+func readableArtifactAttemptIDs(ctx *ArtifactContext, task *actions.ActionTask) ([]int64, bool) {
+	attemptIDs, err := actions.GetArtifactAttemptIDs(ctx, task.Job)
+	if err != nil {
+		log.Error("Error getting readable artifact attempts: %v", err)
+		ctx.HTTPError(http.StatusInternalServerError, "Error getting readable artifact attempts")
+		return nil, false
+	}
+	return attemptIDs, true
+}
+
 func validateArtifactHash(ctx *ArtifactContext, artifactName string) bool {
 	paramHash := ctx.PathParam("artifact_hash")
 	// use artifact name to create upload url

routers/api/actions/artifactsv4.go

@@ -107,7 +107,6 @@ import (
 	actions_module "gitea.dev/modules/actions"
 	"gitea.dev/modules/httplib"
 	"gitea.dev/modules/log"
-	"gitea.dev/modules/optional"
 	"gitea.dev/modules/setting"
 	"gitea.dev/modules/storage"
 	"gitea.dev/modules/util"
@@ -262,9 +261,28 @@ func (r *artifactV4Routes) verifySignature(ctx *ArtifactContext, endp string) (*
 	return task, artifactName, true
 }
 
-func (r *artifactV4Routes) getArtifactByName(ctx *ArtifactContext, runID, runAttemptID int64, name string) (*actions_model.ActionArtifact, error) {
+// getOwnAttemptArtifactByName resolves an artifact of the given attempt whatever its status,
+// since upload and finalize work on the pending row they just created.
+func (r *artifactV4Routes) getOwnAttemptArtifactByName(ctx *ArtifactContext, runID, runAttemptID int64, name string) (*actions_model.ActionArtifact, error) {
+	return r.findArtifactByName(ctx, runID, []int64{runAttemptID}, name, nil)
+}
+
+// getDownloadableArtifactByName resolves the newest artifact with the given name within the attempts whose content can still be served,
+// so a pending, deleted or expired row of a newer attempt does not shadow the confirmed copy inherited from an older one.
+func (r *artifactV4Routes) getDownloadableArtifactByName(ctx *ArtifactContext, runID int64, runAttemptIDs []int64, name string) (*actions_model.ActionArtifact, error) {
+	return r.findArtifactByName(ctx, runID, runAttemptIDs, name, builder.Eq{"status": actions_model.ArtifactStatusUploadConfirmed})
+}
+
+func (r *artifactV4Routes) findArtifactByName(ctx *ArtifactContext, runID int64, runAttemptIDs []int64, name string, extraCond builder.Cond) (*actions_model.ActionArtifact, error) {
+	cond := builder.NewCond().
+		And(builder.Eq{"run_id": runID, "artifact_name": name}, builder.Like{"content_encoding", "%/%"}).
+		And(builder.In("run_attempt_id", runAttemptIDs))
+	if extraCond != nil {
+		cond = cond.And(extraCond)
+	}
+
 	var art actions_model.ActionArtifact
-	has, err := db.GetEngine(ctx).Where(builder.Eq{"run_id": runID, "run_attempt_id": runAttemptID, "artifact_name": name}, builder.Like{"content_encoding", "%/%"}).Get(&art)
+	has, err := db.GetEngine(ctx).Where(cond).OrderBy("run_attempt_id DESC, id DESC").Get(&art)
 	if err != nil {
 		return nil, err
 	} else if !has {
@@ -384,7 +402,7 @@ func (r *artifactV4Routes) uploadArtifact(ctx *ArtifactContext) {
 	switch comp {
 	case "block", "appendBlock":
 		// get artifact by name
-		artifact, err := r.getArtifactByName(ctx, task.Job.RunID, task.Job.RunAttemptID, artifactName)
+		artifact, err := r.getOwnAttemptArtifactByName(ctx, task.Job.RunID, task.Job.RunAttemptID, artifactName)
 		if err != nil {
 			log.Error("Error artifact not found: %v", err)
 			ctx.HTTPError(http.StatusNotFound, "Error artifact not found")
@@ -471,7 +489,7 @@ func (r *artifactV4Routes) finalizeArtifact(ctx *ArtifactContext) {
 	}
 
 	// get artifact by name
-	artifact, err := r.getArtifactByName(ctx, runID, ctx.ActionTask.Job.RunAttemptID, req.Name)
+	artifact, err := r.getOwnAttemptArtifactByName(ctx, runID, ctx.ActionTask.Job.RunAttemptID, req.Name)
 	if err != nil {
 		log.Error("Error artifact not found: %v", err)
 		ctx.HTTPError(http.StatusNotFound, "Error artifact not found")
@@ -578,14 +596,18 @@ func (r *artifactV4Routes) listArtifacts(ctx *ArtifactContext) {
 	if ok := r.parseProtobufBody(ctx, &req); !ok {
 		return
 	}
-	_, runID, ok := validateRunIDV4(ctx, req.WorkflowRunBackendId)
+	task, runID, ok := validateRunIDV4(ctx, req.WorkflowRunBackendId)
+	if !ok {
+		return
+	}
+	attemptIDs, ok := readableArtifactAttemptIDs(ctx, task)
 	if !ok {
 		return
 	}
 
-	artifacts, err := db.Find[actions_model.ActionArtifact](ctx, actions_model.FindArtifactsOptions{
+	artifacts, err := actions_model.FindReadableArtifacts(ctx, actions_model.FindArtifactsOptions{
 		RunID:                runID,
-		RunAttemptID:         optional.Some(ctx.ActionTask.Job.RunAttemptID),
+		RunAttemptIDs:        attemptIDs,
 		Status:               int(actions_model.ArtifactStatusUploadConfirmed),
 		FinalizedArtifactsV4: true,
 	})
@@ -597,6 +619,8 @@ func (r *artifactV4Routes) listArtifacts(ctx *ArtifactContext) {
 
 	list := []*ListArtifactsResponse_MonolithArtifact{}
 
+	// both filters pick from what this attempt may read, so they run after the shadowed artifacts are gone:
+	// a shadowed artifact is not downloadable either, GetSignedArtifactURL resolves by name
 	table := map[string]*ListArtifactsResponse_MonolithArtifact{}
 	for _, artifact := range artifacts {
 		if _, ok := table[artifact.ArtifactName]; ok || req.IdFilter != nil && artifact.ID != req.IdFilter.Value || req.NameFilter != nil && artifact.ArtifactName != req.NameFilter.Value {
@@ -631,7 +655,11 @@ func (r *artifactV4Routes) getSignedArtifactURL(ctx *ArtifactContext) {
 	if ok := r.parseProtobufBody(ctx, &req); !ok {
 		return
 	}
-	_, runID, ok := validateRunIDV4(ctx, req.WorkflowRunBackendId)
+	task, runID, ok := validateRunIDV4(ctx, req.WorkflowRunBackendId)
+	if !ok {
+		return
+	}
+	attemptIDs, ok := readableArtifactAttemptIDs(ctx, task)
 	if !ok {
 		return
 	}
@@ -639,17 +667,12 @@ func (r *artifactV4Routes) getSignedArtifactURL(ctx *ArtifactContext) {
 	artifactName := req.Name
 
 	// get artifact by name
-	artifact, err := r.getArtifactByName(ctx, runID, ctx.ActionTask.Job.RunAttemptID, artifactName)
+	artifact, err := r.getDownloadableArtifactByName(ctx, runID, attemptIDs, artifactName)
 	if err != nil {
 		log.Error("Error artifact not found: %v", err)
 		ctx.HTTPError(http.StatusNotFound, "Error artifact not found")
 		return
 	}
-	if artifact.Status != actions_model.ArtifactStatusUploadConfirmed {
-		log.Error("Error artifact not found: %s", artifact.Status.ToString())
-		ctx.HTTPError(http.StatusNotFound, "Error artifact not found")
-		return
-	}
 
 	respData := GetSignedArtifactURLResponse{}
 
@@ -671,16 +694,15 @@ func (r *artifactV4Routes) downloadArtifact(ctx *ArtifactContext) {
 	if !ok {
 		return
 	}
-
-	// get artifact by name
-	artifact, err := r.getArtifactByName(ctx, task.Job.RunID, task.Job.RunAttemptID, artifactName)
-	if err != nil {
-		log.Error("Error artifact not found: %v", err)
-		ctx.HTTPError(http.StatusNotFound, "Error artifact not found")
+	attemptIDs, ok := readableArtifactAttemptIDs(ctx, task)
+	if !ok {
 		return
 	}
-	if artifact.Status != actions_model.ArtifactStatusUploadConfirmed {
-		log.Error("Error artifact not found: %s", artifact.Status.ToString())
+
+	// get artifact by name
+	artifact, err := r.getDownloadableArtifactByName(ctx, task.Job.RunID, attemptIDs, artifactName)
+	if err != nil {
+		log.Error("Error artifact not found: %v", err)
 		ctx.HTTPError(http.StatusNotFound, "Error artifact not found")
 		return
 	}
@@ -704,7 +726,7 @@ func (r *artifactV4Routes) deleteArtifact(ctx *ArtifactContext) {
 	}
 
 	// get artifact by name
-	artifact, err := r.getArtifactByName(ctx, runID, ctx.ActionTask.Job.RunAttemptID, req.Name)
+	artifact, err := r.getOwnAttemptArtifactByName(ctx, runID, ctx.ActionTask.Job.RunAttemptID, req.Name)
 	if err != nil {
 		log.Error("Error artifact not found: %v", err)
 		ctx.HTTPError(http.StatusNotFound, "Error artifact not found")