mirror of
https://github.com/go-gitea/gitea.git
synced 2026-11-16 16:11:06 +00:00
Backport https://github.com/go-gitea/gitea/pull/40219 levelqueue v0.5.1 fixes this upstream (https://gitea.com/lunny/levelqueue/pulls/8), so this now only bumps the dependency. A push wrote "high" before the payload, so an interrupted push left an id counted with no key. `LPop` then returned `ErrNotFound` without advancing "low" and the wrapper retried forever: nothing popped, no worker started, later items unreachable. All were only fixable by clearing the queue, losing the backlog: - https://github.com/go-gitea/gitea/issues/32006 — mail, 0 workers - https://github.com/go-gitea/gitea/issues/28919 — mirror, "already in queue" - https://github.com/go-gitea/gitea/issues/33862 — archives hanging - https://github.com/go-gitea/gitea/issues/33882 — self-healing request v0.5.1 writes counter and payload in one batch, moves the counters past missing keys, and repairs leftover unique-queue set entries, so affected instances recover on their next pop. No test: the behaviour lives in the dependency, which covers it in its own `recover_test.go`. Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: wxiaoguang <wxiaoguang@gmail.com> Co-authored-by: William Desportes <williamdes@wdes.fr> Co-authored-by: wxiaoguang <wxiaoguang@gmail.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
8 files changed
+30
-152
No files matched your search
@@ -6,13 +6,9 @@ package queue
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.dev/modules/queue/lqinternal"
|
||||
"gitea.dev/modules/setting"
|
||||
|
||||
"gitea.com/lunny/levelqueue"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/syndtr/goleveldb/leveldb"
|
||||
)
|
||||
|
||||
func TestBaseLevelDB(t *testing.T) {
|
||||
@@ -27,53 +23,3 @@ func TestBaseLevelDB(t *testing.T) {
|
||||
testQueueBasic(t, newBaseLevelQueueSimple, toBaseConfig("baseLevelQueue", setting.QueueSettings{Datadir: t.TempDir() + "/queue-test", Length: 10}), optsSimple)
|
||||
testQueueBasic(t, newBaseLevelQueueUnique, toBaseConfig("baseLevelQueueUnique", setting.QueueSettings{ConnStr: "leveldb://" + t.TempDir() + "/queue-test", Length: 10}), optsUnique)
|
||||
}
|
||||
|
||||
func TestCorruptedLevelQueue(t *testing.T) {
|
||||
// sometimes the levelqueue could be in a corrupted state, this test is to make sure it can recover from it
|
||||
dbDir := t.TempDir() + "/levelqueue-test"
|
||||
db, err := leveldb.OpenFile(dbDir, nil)
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
|
||||
assert.NoError(t, db.Put([]byte("other-key"), []byte("other-value"), nil))
|
||||
|
||||
nameQueuePrefix := []byte("queue_name")
|
||||
nameSetPrefix := []byte("set_name")
|
||||
lq, err := levelqueue.NewUniqueQueue(db, nameQueuePrefix, nameSetPrefix, false)
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, lq.RPush([]byte("item-1")))
|
||||
|
||||
itemKey := lqinternal.QueueItemKeyBytes(nameQueuePrefix, 1)
|
||||
itemValue, err := db.Get(itemKey, nil)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, []byte("item-1"), itemValue)
|
||||
|
||||
// there should be 5 keys in db: queue low, queue high, 1 queue item, 1 set item, and "other-key"
|
||||
keys := lqinternal.ListLevelQueueKeys(db)
|
||||
assert.Len(t, keys, 5)
|
||||
|
||||
// delete the queue item key, to corrupt the queue
|
||||
assert.NoError(t, db.Delete(itemKey, nil))
|
||||
// now the queue is corrupted, it never works again
|
||||
_, err = lq.LPop()
|
||||
assert.ErrorIs(t, err, levelqueue.ErrNotFound)
|
||||
assert.NoError(t, lq.Close())
|
||||
|
||||
// remove all the queue related keys to reset the queue
|
||||
lqinternal.RemoveLevelQueueKeys(db, nameQueuePrefix)
|
||||
lqinternal.RemoveLevelQueueKeys(db, nameSetPrefix)
|
||||
// now there should be only 1 key in db: "other-key"
|
||||
keys = lqinternal.ListLevelQueueKeys(db)
|
||||
assert.Len(t, keys, 1)
|
||||
assert.Equal(t, []byte("other-key"), keys[0])
|
||||
|
||||
// re-create a queue from db
|
||||
lq, err = levelqueue.NewUniqueQueue(db, nameQueuePrefix, nameSetPrefix, false)
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, lq.RPush([]byte("item-new-1")))
|
||||
// now the queue works again
|
||||
itemValue, err = lq.LPop()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, []byte("item-new-1"), itemValue)
|
||||
assert.NoError(t, lq.Close())
|
||||
}
|
||||
Reference in new issue
Block a user