mirror of
https://github.com/go-gitea/gitea.git
synced 2026-08-14 03:11:17 +00:00
fix(actions): let a rerun of selected jobs read the previous attempt's artifacts (#38857)
Fixes #38773 ## Background Artifacts became attempt-scoped in #37119, and the runner-facing artifact APIs filter strictly by the attempt of the running job. "Re-run failed jobs" creates a new attempt whose passed-through jobs never upload their artifacts again, so a re-run job that downloads one of them fails with "artifact not found". ## Fix The read paths (v3 and v4 list and download) now resolve artifacts across the running job's attempt plus the attempts it inherits from, and an inherited artifact is shadowed by a same-named one from a newer attempt. ## Note GitHub's documentation does not document these behaviors. The conclusions below are based on manual testing, so consistency with GitHub cannot be guaranteed. - In a "partial re-run", a job can download artifacts uploaded by an earlier attempt, every attempt keeps its own copy of a name, and a lookup by name resolves to the newest one. - A full "Re-run all jobs" never downloads artifacts from earlier attempts. --------- Co-authored-by: bircni <bircni@icloud.com>
This commit is contained in:
Zettat123
@@ -583,6 +583,10 @@ jobs:
|
||||
t.Run("testActionRunAttemptArtifactV4", func(t *testing.T) {
|
||||
testActionRunAttemptArtifactV4(t, repo, session, runner)
|
||||
})
|
||||
|
||||
t.Run("testPartialRerunArtifactInheritance", func(t *testing.T) {
|
||||
testPartialRerunArtifactInheritance(t, user2, token, repo, session, runner)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
@@ -627,6 +631,107 @@ func testActionRunAttemptArtifactV3(t *testing.T, repo *repo_model.Repository, s
|
||||
assert.Equal(t, strings.Repeat("D", 32), sharedContent2)
|
||||
}
|
||||
|
||||
func testPartialRerunArtifactInheritance(t *testing.T, user *user_model.User, token string, repo *repo_model.Repository, session *TestSession, runner *mockRunner) {
|
||||
wfTreePath := ".gitea/workflows/partial-rerun-artifact.yml"
|
||||
wfFileContent := `name: partial-rerun-artifact
|
||||
on:
|
||||
workflow_dispatch:
|
||||
jobs:
|
||||
job1:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- run: echo 'job1'
|
||||
job2:
|
||||
runs-on: ubuntu-latest
|
||||
needs: job1
|
||||
steps:
|
||||
- run: echo 'job2'
|
||||
`
|
||||
opts := getWorkflowCreateFileOptions(user, repo.DefaultBranch, "create "+wfTreePath, wfFileContent)
|
||||
createWorkflowFile(t, token, user.Name, repo.Name, wfTreePath, opts)
|
||||
|
||||
req := NewRequestWithValues(t, "POST", fmt.Sprintf("/%s/%s/actions/run?workflow=%s", repo.OwnerName, repo.Name, "partial-rerun-artifact.yml"), map[string]string{
|
||||
"ref": "refs/heads/main",
|
||||
})
|
||||
session.MakeRequest(t, req, http.StatusSeeOther)
|
||||
|
||||
// job1 uploads the artifacts and succeeds
|
||||
task1 := runner.fetchTask(t)
|
||||
_, job1, run := getTaskAndJobAndRunByTaskID(t, task1.Id)
|
||||
taskToken1 := task1.Context.GetFields()["gitea_runtime_token"].GetStringValue()
|
||||
uploadTestArtifactFileV4(t, run.ID, job1.ID, taskToken1, "job1-only", strings.Repeat("A", 32))
|
||||
uploadTestArtifactFileV4(t, run.ID, job1.ID, taskToken1, "job1-shared", strings.Repeat("B", 32))
|
||||
// a v3 artifact is one row per file, so this one spans two rows
|
||||
uploadTestArtifactFile(t, run.ID, taskToken1, "job1-v3", "a.txt", strings.Repeat("D", 32))
|
||||
uploadTestArtifactFile(t, run.ID, taskToken1, "job1-v3", "b.txt", strings.Repeat("E", 32))
|
||||
runner.execTask(t, task1, &mockTaskOutcome{result: runnerv1.Result_RESULT_SUCCESS})
|
||||
|
||||
// job2 fails
|
||||
task2 := runner.fetchTask(t)
|
||||
_, job2, _ := getTaskAndJobAndRunByTaskID(t, task2.Id)
|
||||
runner.execTask(t, task2, &mockTaskOutcome{result: runnerv1.Result_RESULT_FAILURE})
|
||||
|
||||
// re-run only the failed job, so job1 is passed through and never uploads its artifacts again
|
||||
req = NewRequest(t, "POST", fmt.Sprintf("/%s/%s/actions/runs/%d/rerun-failed", repo.OwnerName, repo.Name, run.ID))
|
||||
session.MakeRequest(t, req, http.StatusOK)
|
||||
|
||||
task3 := runner.fetchTask(t)
|
||||
_, job3, _ := getTaskAndJobAndRunByTaskID(t, task3.Id)
|
||||
require.Equal(t, job2.JobID, job3.JobID)
|
||||
require.NotEqual(t, job2.RunAttemptID, job3.RunAttemptID)
|
||||
taskToken3 := task3.Context.GetFields()["gitea_runtime_token"].GetStringValue()
|
||||
|
||||
// the new attempt inherits what the previous attempt uploaded
|
||||
assert.ElementsMatch(t, []string{"job1-only", "job1-shared"}, listArtifactNamesForRunV4(t, run.ID, job3.ID, taskToken3))
|
||||
assert.Equal(t, strings.Repeat("A", 32), downloadArtifactContentV4ByTask(t, run.ID, job3.ID, taskToken3, "job1-only"))
|
||||
assert.Contains(t, listArtifactNamesForRun(t, run.ID, taskToken3), "job1-v3")
|
||||
|
||||
// a pending upload of this attempt must not shadow the confirmed copy it inherited
|
||||
createTestArtifactV4(t, run.ID, job3.ID, taskToken3, "job1-only")
|
||||
assert.Equal(t, strings.Repeat("A", 32), downloadArtifactContentV4ByTask(t, run.ID, job3.ID, taskToken3, "job1-only"))
|
||||
|
||||
// both rows of the inherited v3 artifact are readable
|
||||
inheritedV3 := getArtifactDownloadItemsForRun(t, run.ID, taskToken3, "job1-v3")
|
||||
require.Len(t, inheritedV3, 2)
|
||||
assert.Equal(t, strings.Repeat("D", 32), downloadArtifactItemContent(t, taskToken3, inheritedV3[0]))
|
||||
|
||||
// uploading an inherited name in this attempt shadows the inherited artifact
|
||||
inheritedSharedID := listArtifactIDForRunV4(t, run.ID, job3.ID, taskToken3, "job1-shared")
|
||||
require.Len(t, listArtifactsByIDV4(t, run.ID, job3.ID, inheritedSharedID, taskToken3), 1)
|
||||
uploadTestArtifactFileV4(t, run.ID, job3.ID, taskToken3, "job1-shared", strings.Repeat("C", 32))
|
||||
assert.ElementsMatch(t, []string{"job1-only", "job1-shared"}, listArtifactNamesForRunV4(t, run.ID, job3.ID, taskToken3))
|
||||
assert.Equal(t, strings.Repeat("C", 32), downloadArtifactContentV4ByTask(t, run.ID, job3.ID, taskToken3, "job1-shared"))
|
||||
|
||||
// a shadowed artifact is not listed by id either: a download resolves by name
|
||||
assert.Empty(t, listArtifactsByIDV4(t, run.ID, job3.ID, inheritedSharedID, taskToken3))
|
||||
|
||||
// the shadowed v3 artifact is dropped as a whole, its b.txt row must not survive next to the new a.txt
|
||||
uploadTestArtifactFile(t, run.ID, taskToken3, "job1-v3", "a.txt", strings.Repeat("F", 32))
|
||||
shadowedV3 := getArtifactDownloadItemsForRun(t, run.ID, taskToken3, "job1-v3")
|
||||
require.Len(t, shadowedV3, 1)
|
||||
assert.Equal(t, strings.Repeat("F", 32), downloadArtifactItemContent(t, taskToken3, shadowedV3[0]))
|
||||
|
||||
runner.execTask(t, task3, &mockTaskOutcome{result: runnerv1.Result_RESULT_SUCCESS})
|
||||
}
|
||||
|
||||
func getArtifactDownloadItemsForRun(t *testing.T, runID int64, taskToken, artifactName string) []downloadArtifactResponseItem {
|
||||
t.Helper()
|
||||
|
||||
req := NewRequest(t, "GET", fmt.Sprintf("/api/actions_pipeline/_apis/pipelines/workflows/%d/artifacts/%x/download_url?itemPath=%s", runID, md5.Sum([]byte(artifactName)), artifactName)).
|
||||
AddTokenAuth(taskToken)
|
||||
resp := MakeRequest(t, req, http.StatusOK)
|
||||
return DecodeJSON(t, resp, &downloadArtifactResponse{}).Value
|
||||
}
|
||||
|
||||
func downloadArtifactItemContent(t *testing.T, taskToken string, item downloadArtifactResponseItem) string {
|
||||
t.Helper()
|
||||
|
||||
idx := strings.Index(item.ContentLocation, "/api/actions_pipeline/_apis/pipelines/")
|
||||
require.NotEqual(t, -1, idx)
|
||||
req := NewRequest(t, "GET", item.ContentLocation[idx:]).AddTokenAuth(taskToken)
|
||||
return MakeRequest(t, req, http.StatusOK).Body.String()
|
||||
}
|
||||
|
||||
func uploadTestArtifactFile(t *testing.T, runID int64, authToken, artifactName, fileName, content string) {
|
||||
t.Helper()
|
||||
|
||||
|
||||
@@ -947,7 +947,24 @@ func testActionRunAttemptArtifactV4(t *testing.T, repo *repo_model.Repository, s
|
||||
assert.Equal(t, strings.Repeat("D", 32), downloadRepoArtifactV4Content(t, session, sharedArtifactsResp.Entries[1].ArchiveDownloadURL))
|
||||
}
|
||||
|
||||
func uploadTestArtifactFileV4(t *testing.T, runID, jobID int64, authToken, artifactName, content string) {
|
||||
func downloadArtifactContentV4ByTask(t *testing.T, runID, jobID int64, taskToken, artifactName string) string {
|
||||
t.Helper()
|
||||
|
||||
req := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/GetSignedArtifactURL", toProtoJSON(&actions.GetSignedArtifactURLRequest{
|
||||
Name: artifactName,
|
||||
WorkflowRunBackendId: strconv.FormatInt(runID, 10),
|
||||
WorkflowJobRunBackendId: strconv.FormatInt(jobID, 10),
|
||||
})).AddTokenAuth(taskToken)
|
||||
resp := MakeRequest(t, req, http.StatusOK)
|
||||
var urlResp actions.GetSignedArtifactURLResponse
|
||||
require.NoError(t, protojson.Unmarshal(resp.Body.Bytes(), &urlResp))
|
||||
require.NotEmpty(t, urlResp.SignedUrl)
|
||||
|
||||
return MakeRequest(t, NewRequest(t, "GET", urlResp.SignedUrl), http.StatusOK).Body.String()
|
||||
}
|
||||
|
||||
// createTestArtifactV4 only creates the artifact record, leaving it pending until it is uploaded and finalized
|
||||
func createTestArtifactV4(t *testing.T, runID, jobID int64, authToken, artifactName string) string {
|
||||
t.Helper()
|
||||
|
||||
req := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/CreateArtifact", toProtoJSON(&actions.CreateArtifactRequest{
|
||||
@@ -958,11 +975,17 @@ func uploadTestArtifactFileV4(t *testing.T, runID, jobID int64, authToken, artif
|
||||
MimeType: wrapperspb.String("application/zip"),
|
||||
})).AddTokenAuth(authToken)
|
||||
resp := MakeRequest(t, req, http.StatusOK)
|
||||
var uploadResp actions.CreateArtifactResponse
|
||||
require.NoError(t, protojson.Unmarshal(resp.Body.Bytes(), &uploadResp))
|
||||
require.True(t, uploadResp.Ok)
|
||||
var createResp actions.CreateArtifactResponse
|
||||
require.NoError(t, protojson.Unmarshal(resp.Body.Bytes(), &createResp))
|
||||
require.True(t, createResp.Ok)
|
||||
return createResp.SignedUploadUrl
|
||||
}
|
||||
|
||||
req = NewRequestWithBody(t, "PUT", uploadResp.SignedUploadUrl+"&comp=appendBlock", strings.NewReader(content))
|
||||
func uploadTestArtifactFileV4(t *testing.T, runID, jobID int64, authToken, artifactName, content string) {
|
||||
t.Helper()
|
||||
|
||||
signedUploadURL := createTestArtifactV4(t, runID, jobID, authToken, artifactName)
|
||||
req := NewRequestWithBody(t, "PUT", signedUploadURL+"&comp=appendBlock", strings.NewReader(content))
|
||||
MakeRequest(t, req, http.StatusCreated)
|
||||
|
||||
sum := sha256.Sum256([]byte(content))
|
||||
@@ -973,30 +996,59 @@ func uploadTestArtifactFileV4(t *testing.T, runID, jobID int64, authToken, artif
|
||||
WorkflowRunBackendId: strconv.FormatInt(runID, 10),
|
||||
WorkflowJobRunBackendId: strconv.FormatInt(jobID, 10),
|
||||
})).AddTokenAuth(authToken)
|
||||
resp = MakeRequest(t, req, http.StatusOK)
|
||||
resp := MakeRequest(t, req, http.StatusOK)
|
||||
var finalizeResp actions.FinalizeArtifactResponse
|
||||
require.NoError(t, protojson.Unmarshal(resp.Body.Bytes(), &finalizeResp))
|
||||
require.True(t, finalizeResp.Ok)
|
||||
}
|
||||
|
||||
func listArtifactsForRunV4(t *testing.T, taskToken string, req *actions.ListArtifactsRequest) []*actions.ListArtifactsResponse_MonolithArtifact {
|
||||
t.Helper()
|
||||
|
||||
httpReq := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/ListArtifacts", toProtoJSON(req)).AddTokenAuth(taskToken)
|
||||
resp := MakeRequest(t, httpReq, http.StatusOK)
|
||||
var listResp actions.ListArtifactsResponse
|
||||
require.NoError(t, protojson.Unmarshal(resp.Body.Bytes(), &listResp))
|
||||
return listResp.Artifacts
|
||||
}
|
||||
|
||||
func listArtifactNamesForRunV4(t *testing.T, runID, jobID int64, taskToken string) []string {
|
||||
t.Helper()
|
||||
|
||||
req := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/ListArtifacts", toProtoJSON(&actions.ListArtifactsRequest{
|
||||
artifacts := listArtifactsForRunV4(t, taskToken, &actions.ListArtifactsRequest{
|
||||
WorkflowRunBackendId: strconv.FormatInt(runID, 10),
|
||||
WorkflowJobRunBackendId: strconv.FormatInt(jobID, 10),
|
||||
})).AddTokenAuth(taskToken)
|
||||
resp := MakeRequest(t, req, http.StatusOK)
|
||||
var listResp actions.ListArtifactsResponse
|
||||
require.NoError(t, protojson.Unmarshal(resp.Body.Bytes(), &listResp))
|
||||
})
|
||||
|
||||
names := make([]string, 0, len(listResp.Artifacts))
|
||||
for _, item := range listResp.Artifacts {
|
||||
names := make([]string, 0, len(artifacts))
|
||||
for _, item := range artifacts {
|
||||
names = append(names, item.Name)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func listArtifactIDForRunV4(t *testing.T, runID, jobID int64, taskToken, artifactName string) int64 {
|
||||
t.Helper()
|
||||
|
||||
artifacts := listArtifactsForRunV4(t, taskToken, &actions.ListArtifactsRequest{
|
||||
NameFilter: wrapperspb.String(artifactName),
|
||||
WorkflowRunBackendId: strconv.FormatInt(runID, 10),
|
||||
WorkflowJobRunBackendId: strconv.FormatInt(jobID, 10),
|
||||
})
|
||||
require.Len(t, artifacts, 1)
|
||||
return artifacts[0].DatabaseId
|
||||
}
|
||||
|
||||
func listArtifactsByIDV4(t *testing.T, runID, jobID, artifactID int64, taskToken string) []*actions.ListArtifactsResponse_MonolithArtifact {
|
||||
t.Helper()
|
||||
|
||||
return listArtifactsForRunV4(t, taskToken, &actions.ListArtifactsRequest{
|
||||
IdFilter: wrapperspb.Int64(artifactID),
|
||||
WorkflowRunBackendId: strconv.FormatInt(runID, 10),
|
||||
WorkflowJobRunBackendId: strconv.FormatInt(jobID, 10),
|
||||
})
|
||||
}
|
||||
|
||||
func downloadRepoArtifactV4Content(t *testing.T, session *TestSession, archiveDownloadURL string) string {
|
||||
t.Helper()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user