mirror of
https://github.com/go-gitea/gitea.git
synced 2026-08-19 20:28:34 +00:00
ci: improve caching (#38958)
- only `cache-seeder` writes caches, every other workflow restores. Saves were being rejected once the repo went over its cache budget, leaving main's caches stale and PR runs building cold - seed the pnpm store and uv caches next to the go ones, so PRs warm-start on them rather than installing from scratch - prune keeps a single generation per key, including across go versions, where a toolchain bump leaves the previous build cache unusable. Reclaims ~2.6 GB immediately - prune runs every 6h instead of daily and trims to 6 GB, since CodeQL writes ~200 MB per push to main from outside this repo's workflows - pull requests and release branches no longer write pnpm, uv and binfmt caches, whose ref-scoped copies are never read again --------- Signed-off-by: silverwind <me@silverwind.io>
This commit is contained in:
@@ -10,6 +10,8 @@ runs:
|
||||
using: composite
|
||||
steps:
|
||||
- uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
|
||||
with:
|
||||
cache-image: false
|
||||
- uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
- name: Build regular image
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
|
||||
@@ -31,7 +31,7 @@ runs:
|
||||
with:
|
||||
path: ~/go/pkg/mod
|
||||
key: gomod-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('go.sum') }}
|
||||
restore-keys: gomod-${{ runner.os }}-${{ runner.arch }}
|
||||
restore-keys: gomod-${{ runner.os }}-${{ runner.arch }}-
|
||||
- if: ${{ github.workflow == 'cache-seeder' && inputs.lint-cache != 'true' }}
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
|
||||
@@ -1,22 +1,30 @@
|
||||
name: node-setup
|
||||
description: Set up pnpm and node and restore caches
|
||||
description: Set up pnpm and node and restore the pnpm store cache
|
||||
|
||||
inputs:
|
||||
cache:
|
||||
description: Cache pnpm downloads
|
||||
description: Restore the pnpm store cache
|
||||
default: "true"
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: 26
|
||||
- if: ${{ inputs.cache == 'true' }}
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
id: store
|
||||
shell: bash
|
||||
run: echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
|
||||
- if: ${{ inputs.cache == 'true' && github.workflow == 'cache-seeder' }}
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
node-version: 26
|
||||
cache: pnpm
|
||||
cache-dependency-path: pnpm-lock.yaml
|
||||
- if: ${{ inputs.cache != 'true' }}
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
path: ${{ steps.store.outputs.path }}
|
||||
key: pnpm-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('pnpm-lock.yaml') }}
|
||||
- if: ${{ inputs.cache == 'true' && github.workflow != 'cache-seeder' }}
|
||||
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
node-version: 26
|
||||
path: ${{ steps.store.outputs.path }}
|
||||
key: pnpm-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('pnpm-lock.yaml') }}
|
||||
restore-keys: pnpm-${{ runner.os }}-${{ runner.arch }}-
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
name: python-setup
|
||||
description: Set up uv and python and restore the uv cache
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- uses: astral-sh/setup-uv@ae62891fec2bb8e7d6c99fc78c9fec3a63790f8d # v10.0.0
|
||||
with:
|
||||
python-version: 3.14
|
||||
save-cache: ${{ github.workflow == 'cache-seeder' }}
|
||||
@@ -5,7 +5,7 @@ name: cache-prune
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "37 2 * * *" # every day at 02:37 UTC
|
||||
- cron: "37 */6 * * *" # every six hours at :37
|
||||
workflow_dispatch:
|
||||
workflow_call:
|
||||
|
||||
@@ -24,15 +24,25 @@ jobs:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
steps:
|
||||
# Deletes least recently used first, the order GitHub itself evicts in, which takes
|
||||
# superseded generations first as those stop being restored once a newer one exists.
|
||||
# Keep the newest generation per key, restores never reach the older ones.
|
||||
- name: delete superseded caches
|
||||
run: |
|
||||
gh cache list --limit 1000 --json id,key,ref,createdAt |
|
||||
jq -r 'group_by([.ref, (.key | sub("(-go[0-9.]+)?-[0-9a-f]{40,64}(-[0-9]+-[0-9]+)?$"; ""))])[]
|
||||
| sort_by(.createdAt)[:-1][] | "\(.id) \(.key)"' |
|
||||
while read -r id key; do
|
||||
echo "deleting $key"
|
||||
gh cache delete "$id" || true
|
||||
done
|
||||
|
||||
# Deletes least recently used first, the order GitHub itself evicts in.
|
||||
- name: delete caches over the size limit
|
||||
run: |
|
||||
caches=$(gh cache list --limit 1000 --sort last_accessed_at --order asc --json id,key,sizeInBytes)
|
||||
size=$(jq '[.[].sizeInBytes] | add // 0' <<< "$caches")
|
||||
echo "cache usage: $((size / 1000000)) MB"
|
||||
while [ "$size" -gt 6500000000 ] && read -r id bytes key; do
|
||||
while [ "$size" -gt 6000000000 ] && read -r id bytes key; do
|
||||
echo "deleting $key"
|
||||
gh cache delete "$id"
|
||||
gh cache delete "$id" || true
|
||||
size=$((size - bytes))
|
||||
done <<< "$(jq -r '.[] | "\(.id) \(.sizeInBytes) \(.key)"' <<< "$caches")"
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
# Populates main's cache scope so PR runs warm-start from it. Saves the go
|
||||
# module, go build (incl. test compile), and golangci-lint caches.
|
||||
#
|
||||
# Caches are ref-scoped: PR runs read their own scope then fall back to the
|
||||
# base branch. Per .github/actions/go-cache/action.yml, PRs are restore-only,
|
||||
# so push-to-main is the only opportunity to populate the fallback scope.
|
||||
# Populates main's cache scope so PR runs warm-start from it. Caches are ref-scoped:
|
||||
# PR runs read their own scope then fall back to the base branch, and only this
|
||||
# workflow saves, so push-to-main is the only chance to populate the fallback scope.
|
||||
|
||||
name: cache-seeder
|
||||
|
||||
@@ -16,8 +13,13 @@ on:
|
||||
- "go.mod" # a toolchain bump invalidates the build caches
|
||||
- "go.sum"
|
||||
- ".golangci.yml"
|
||||
- "pnpm-lock.yaml"
|
||||
- "pyproject.toml"
|
||||
- "uv.lock"
|
||||
- ".github/actions/go-cache/action.yml"
|
||||
- ".github/actions/go-setup/action.yml"
|
||||
- ".github/actions/node-setup/action.yml"
|
||||
- ".github/actions/python-setup/action.yml"
|
||||
- ".github/workflows/cache-seeder.yml"
|
||||
|
||||
concurrency:
|
||||
@@ -74,9 +76,23 @@ jobs:
|
||||
TAGS: ${{ matrix.tags }}
|
||||
TARGET: ${{ matrix.target }}
|
||||
|
||||
frontend:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: ./.github/actions/node-setup
|
||||
- run: make deps-frontend
|
||||
|
||||
python:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: ./.github/actions/python-setup
|
||||
- run: make deps-py
|
||||
|
||||
# reclaims the caches this run superseded, so the next save still fits in the allowance
|
||||
prune:
|
||||
needs: [gobuild, lint]
|
||||
needs: [gobuild, lint, frontend, python]
|
||||
permissions:
|
||||
actions: write
|
||||
uses: ./.github/workflows/cache-prune.yml
|
||||
|
||||
@@ -42,9 +42,7 @@ jobs:
|
||||
- run: make lint-spell
|
||||
|
||||
- if: needs.files-changed.outputs.templates == 'true' || needs.files-changed.outputs.yaml == 'true' || needs.files-changed.outputs.actions == 'true'
|
||||
uses: astral-sh/setup-uv@ae62891fec2bb8e7d6c99fc78c9fec3a63790f8d # v10.0.0
|
||||
with:
|
||||
python-version: 3.14
|
||||
uses: ./.github/actions/python-setup
|
||||
- if: needs.files-changed.outputs.templates == 'true' || needs.files-changed.outputs.yaml == 'true'
|
||||
run: make deps-py lint-templates lint-yaml
|
||||
|
||||
|
||||
@@ -80,6 +80,8 @@ jobs:
|
||||
# fetch all tags to ensure that "git describe" reports expected Gitea version, eg. v1.21.0-dev-1-g1234567
|
||||
- run: git fetch --unshallow --quiet --tags --force
|
||||
- uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
|
||||
with:
|
||||
cache-image: false
|
||||
- uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
- name: Get cleaned branch name
|
||||
id: clean_name
|
||||
|
||||
@@ -91,6 +91,8 @@ jobs:
|
||||
# fetch all tags to ensure that "git describe" reports expected Gitea version, eg. v1.21.0-dev-1-g1234567
|
||||
- run: git fetch --unshallow --quiet --tags --force
|
||||
- uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
|
||||
with:
|
||||
cache-image: false
|
||||
- uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
- uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
|
||||
id: meta
|
||||
|
||||
@@ -94,6 +94,8 @@ jobs:
|
||||
# fetch all tags to ensure that "git describe" reports expected Gitea version, eg. v1.21.0-dev-1-g1234567
|
||||
- run: git fetch --unshallow --quiet --tags --force
|
||||
- uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
|
||||
with:
|
||||
cache-image: false
|
||||
- uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
- uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
|
||||
id: meta
|
||||
|
||||
Reference in New Issue
Block a user