perf(actions): debounce runner heartbeat writes and throttle task picks (#38281)

3 reductions in the DB load generated by many runners polling `FetchTask`: **1. Debounce runner heartbeat writes** Every poll wrote `last_online`, and every `UpdateTask`/`UpdateLog` wrote `last_active` — while a runner streams logs that is many writes per second per runner. These are now persisted only when stale enough to actually affect the active/offline status (`ShouldPersistLastOnline` / `ShouldPersistLastActive`), using the existing columns. **2. Throttle concurrent task picks** A new in-process semaphore (`MAX_CONCURRENT_TASK_PICKS`) bounds how many runners run the task-assignment transaction at once, so a fleet polling together cannot stampede the query. Throttled polls retry on their next poll without advancing the runner's tasks version. **3. Paginate the task-pick query** `CreateTaskForRunner` previously loaded every waiting job in the runner's scope into memory on each poll (no `LIMIT`). Now it pages through the waiting backlog oldest-first with `LIMIT`, claiming the first label-matching job. --------- Co-authored-by: Zettat123 <zettat123@gmail.com>
2026-08-02 22:39:36 +00:00 · 2026-07-07 21:16:20 +02:00
parent 97078b96cf
commit 308a6f12ae
10 changed files with 317 additions and 30 deletions
@@ -371,3 +371,74 @@ func TestReleaseTaskForRunner(t *testing.T) {
 	unittest.AssertNotExistsBean(t, &ActionTask{ID: task.ID})
 	unittest.AssertNotExistsBean(t, &ActionTaskStep{TaskID: task.ID})
 }
+
+// TestCreateTaskForRunnerPagination verifies that a job sitting beyond the first page is still claimed
+func TestCreateTaskForRunnerPagination(t *testing.T) {
+	require.NoError(t, unittest.PrepareTestDatabase())
+
+	defer func(orig int) { pickTaskBatchSize = orig }(pickTaskBatchSize)
+	pickTaskBatchSize = 2
+
+	run := &ActionRun{
+		Title:         "pagination-test-run",
+		RepoID:        1,
+		OwnerID:       2,
+		WorkflowID:    "test.yaml",
+		Index:         9903,
+		TriggerUserID: 2,
+		Ref:           "refs/heads/main",
+		CommitSHA:     "c2d72f548424103f01ee1dc02889c1e2bff816b0",
+		Event:         "push",
+		TriggerEvent:  "push",
+		Status:        StatusWaiting,
+	}
+	require.NoError(t, db.Insert(t.Context(), run))
+
+	// Five waiting jobs the runner cannot run, then one it can.
+	// With a page size of 2 the matching job only appears on the third page.
+	for i := range 5 {
+		mismatch := &ActionRunJob{
+			RunID:     run.ID,
+			RepoID:    run.RepoID,
+			OwnerID:   run.OwnerID,
+			CommitSHA: run.CommitSHA,
+			Name:      "mismatch-" + string(rune('a'+i)),
+			Attempt:   1,
+			JobID:     "mismatch-" + string(rune('a'+i)),
+			Status:    StatusWaiting,
+			RunsOn:    []string{"windows-latest"},
+		}
+		require.NoError(t, db.Insert(t.Context(), mismatch))
+	}
+
+	target := &ActionRunJob{
+		RunID:           run.ID,
+		RepoID:          run.RepoID,
+		OwnerID:         run.OwnerID,
+		CommitSHA:       run.CommitSHA,
+		Name:            "target-job",
+		Attempt:         1,
+		JobID:           "target-job",
+		Status:          StatusWaiting,
+		RunsOn:          []string{"ubuntu-latest"},
+		WorkflowPayload: []byte("on: push\njobs:\n  target-job:\n    runs-on: ubuntu-latest\n    steps:\n      - run: echo hi\n"),
+	}
+	require.NoError(t, db.Insert(t.Context(), target))
+
+	runner := &ActionRunner{
+		UUID:        "pagination-runner-uuid",
+		Name:        "pagination-runner",
+		AgentLabels: []string{"ubuntu-latest"},
+	}
+	runner.GenerateAndFillToken()
+	require.NoError(t, db.Insert(t.Context(), runner))
+
+	task, ok, err := CreateTaskForRunner(t.Context(), runner)
+	require.NoError(t, err)
+	require.True(t, ok)
+	require.NotNil(t, task)
+
+	claimed := unittest.AssertExistsAndLoadBean(t, &ActionRunJob{ID: target.ID})
+	assert.Equal(t, StatusRunning, claimed.Status)
+	assert.Equal(t, task.ID, claimed.TaskID)
+}