fix(queue): bump levelqueue to v0.5.1 and remove workarounds (#40219) (#40350)

Backport https://github.com/go-gitea/gitea/pull/40219

levelqueue v0.5.1 fixes this upstream
(https://gitea.com/lunny/levelqueue/pulls/8), so this now only bumps the
dependency.

A push wrote "high" before the payload, so an interrupted push left an
id counted with no key. `LPop` then returned `ErrNotFound` without
advancing "low" and the wrapper retried forever: nothing popped, no
worker started, later items unreachable. All were only fixable by
clearing the queue, losing the backlog:
- https://github.com/go-gitea/gitea/issues/32006 — mail, 0 workers
- https://github.com/go-gitea/gitea/issues/28919 — mirror, "already in
queue"
- https://github.com/go-gitea/gitea/issues/33862 — archives hanging
- https://github.com/go-gitea/gitea/issues/33882 — self-healing request

v0.5.1 writes counter and payload in one batch, moves the counters past
missing keys, and repairs leftover unique-queue set entries, so affected
instances recover on their next pop.

No test: the behaviour lives in the dependency, which covers it in its
own `recover_test.go`.

Assisted-by: Claude Code:claude-opus-5-5

Signed-off-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: William Desportes <williamdes@wdes.fr>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-11-13 17:12:25 +00:00
1 parent b60da6d876
commit 95b69a4a2d
8 files changed
+30 -152

No files matched your search

+6 -17
View File
@@ -5,10 +5,8 @@ package queue
import (
"context"
"sync/atomic"
"gitea.dev/modules/nosql"
"gitea.dev/modules/queue/lqinternal"
"gitea.com/lunny/levelqueue"
"github.com/syndtr/goleveldb/leveldb"
@@ -16,7 +14,7 @@ import (
type baseLevelQueue struct {
*baseLevelQueueCommonImpl
internal atomic.Pointer[levelqueue.Queue]
internal *levelqueue.Queue
conn string
cfg *BaseConfig
@@ -41,12 +39,11 @@ func newBaseLevelQueueSimple(cfg *BaseConfig) (baseQueue, error) {
return nil, err
}
q := &baseLevelQueue{conn: conn, cfg: cfg, db: db}
lq, err := levelqueue.NewQueue(db, []byte(cfg.QueueFullName), false)
q.internal, err = levelqueue.NewQueue(db, []byte(cfg.QueueFullName), false)
if err != nil {
return nil, err
}
q.internal.Store(lq)
q.baseLevelQueueCommonImpl = baseLevelQueueCommon(q.cfg, nil, func() baseLevelQueuePushPoper { return q.internal.Load() })
q.baseLevelQueueCommonImpl = baseLevelQueueCommon(q.cfg, nil, q.internal)
return q, nil
}
@@ -55,24 +52,16 @@ func (q *baseLevelQueue) HasItem(ctx context.Context, data []byte) (bool, error)
}
func (q *baseLevelQueue) Len(ctx context.Context) (int, error) {
return int(q.internal.Load().Len()), nil
return int(q.internal.Len()), nil
}
func (q *baseLevelQueue) Close() error {
err := q.internal.Load().Close()
err := q.internal.Close()
_ = nosql.GetManager().CloseLevelDB(q.conn)
q.db = nil // the db is not managed by us, it's managed by the nosql manager
return err
}
func (q *baseLevelQueue) RemoveAll(ctx context.Context) error {
lqinternal.RemoveLevelQueueKeys(q.db, []byte(q.cfg.QueueFullName))
lq, err := levelqueue.NewQueue(q.db, []byte(q.cfg.QueueFullName), false)
if err != nil {
return err
}
old := q.internal.Load()
q.internal.Store(lq)
_ = old.Close() // Not ideal for concurrency. Luckily, the levelqueue only sets its db=nil because it doesn't manage the db, so far so good
return nil
return q.internal.RemoveAll()
}