fix(queue): bump levelqueue to v0.5.1 and remove workarounds (#40219) (#40350)

Backport https://github.com/go-gitea/gitea/pull/40219

levelqueue v0.5.1 fixes this upstream
(https://gitea.com/lunny/levelqueue/pulls/8), so this now only bumps the
dependency.

A push wrote "high" before the payload, so an interrupted push left an
id counted with no key. `LPop` then returned `ErrNotFound` without
advancing "low" and the wrapper retried forever: nothing popped, no
worker started, later items unreachable. All were only fixable by
clearing the queue, losing the backlog:
- https://github.com/go-gitea/gitea/issues/32006 — mail, 0 workers
- https://github.com/go-gitea/gitea/issues/28919 — mirror, "already in
queue"
- https://github.com/go-gitea/gitea/issues/33862 — archives hanging
- https://github.com/go-gitea/gitea/issues/33882 — self-healing request

v0.5.1 writes counter and payload in one batch, moves the counters past
missing keys, and repairs leftover unique-queue set entries, so affected
instances recover on their next pop.

No test: the behaviour lives in the dependency, which covers it in its
own `recover_test.go`.

Assisted-by: Claude Code:claude-opus-5-5

Signed-off-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: William Desportes <williamdes@wdes.fr>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-11-13 17:12:25 +00:00
1 parent b60da6d876
commit 95b69a4a2d
8 files changed
+30 -152

No files matched your search

-54
View File
@@ -6,13 +6,9 @@ package queue
import (
"testing"
"gitea.dev/modules/queue/lqinternal"
"gitea.dev/modules/setting"
"gitea.com/lunny/levelqueue"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/syndtr/goleveldb/leveldb"
)
func TestBaseLevelDB(t *testing.T) {
@@ -27,53 +23,3 @@ func TestBaseLevelDB(t *testing.T) {
testQueueBasic(t, newBaseLevelQueueSimple, toBaseConfig("baseLevelQueue", setting.QueueSettings{Datadir: t.TempDir() + "/queue-test", Length: 10}), optsSimple)
testQueueBasic(t, newBaseLevelQueueUnique, toBaseConfig("baseLevelQueueUnique", setting.QueueSettings{ConnStr: "leveldb://" + t.TempDir() + "/queue-test", Length: 10}), optsUnique)
}
func TestCorruptedLevelQueue(t *testing.T) {
// sometimes the levelqueue could be in a corrupted state, this test is to make sure it can recover from it
dbDir := t.TempDir() + "/levelqueue-test"
db, err := leveldb.OpenFile(dbDir, nil)
require.NoError(t, err)
defer db.Close()
assert.NoError(t, db.Put([]byte("other-key"), []byte("other-value"), nil))
nameQueuePrefix := []byte("queue_name")
nameSetPrefix := []byte("set_name")
lq, err := levelqueue.NewUniqueQueue(db, nameQueuePrefix, nameSetPrefix, false)
assert.NoError(t, err)
assert.NoError(t, lq.RPush([]byte("item-1")))
itemKey := lqinternal.QueueItemKeyBytes(nameQueuePrefix, 1)
itemValue, err := db.Get(itemKey, nil)
assert.NoError(t, err)
assert.Equal(t, []byte("item-1"), itemValue)
// there should be 5 keys in db: queue low, queue high, 1 queue item, 1 set item, and "other-key"
keys := lqinternal.ListLevelQueueKeys(db)
assert.Len(t, keys, 5)
// delete the queue item key, to corrupt the queue
assert.NoError(t, db.Delete(itemKey, nil))
// now the queue is corrupted, it never works again
_, err = lq.LPop()
assert.ErrorIs(t, err, levelqueue.ErrNotFound)
assert.NoError(t, lq.Close())
// remove all the queue related keys to reset the queue
lqinternal.RemoveLevelQueueKeys(db, nameQueuePrefix)
lqinternal.RemoveLevelQueueKeys(db, nameSetPrefix)
// now there should be only 1 key in db: "other-key"
keys = lqinternal.ListLevelQueueKeys(db)
assert.Len(t, keys, 1)
assert.Equal(t, []byte("other-key"), keys[0])
// re-create a queue from db
lq, err = levelqueue.NewUniqueQueue(db, nameQueuePrefix, nameSetPrefix, false)
assert.NoError(t, err)
assert.NoError(t, lq.RPush([]byte("item-new-1")))
// now the queue works again
itemValue, err = lq.LPop()
assert.NoError(t, err)
assert.Equal(t, []byte("item-new-1"), itemValue)
assert.NoError(t, lq.Close())
}