fix(db): make paginated database reads always require "order" option (#39017)

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

models/db/iterate.go

@@ -5,39 +5,72 @@ package db
 
 import (
 	"context"
+	"fmt"
 
 	"gitea.dev/modules/setting"
 
 	"xorm.io/builder"
+	"xorm.io/xorm/schemas"
 )
 
-// Iterate iterates all the Bean object
-func Iterate[Bean any](ctx context.Context, cond builder.Cond, f func(ctx context.Context, bean *Bean) error) error {
-	var start int
-	batchSize := setting.Database.IterateBufferSize
-	sess := GetEngine(ctx)
-	for {
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		default:
-			beans := make([]*Bean, 0, batchSize)
-			if cond != nil {
-				sess = sess.Where(cond)
-			}
-			if err := sess.Limit(batchSize, start).Find(&beans); err != nil {
-				return err
-			}
-			if len(beans) == 0 {
-				return nil
-			}
-			start += len(beans)
+func iterateTableByColumn[Bean any](ctx context.Context, colName string, cond builder.Cond, f func(ctx context.Context, bean *Bean) error) error {
+	table, err := xormEngine.TableInfo(new(Bean))
+	if err != nil {
+		return err
+	}
 
-			for _, bean := range beans {
-				if err := f(ctx, bean); err != nil {
-					return err
-				}
+	var col *schemas.Column
+	if colName == "" {
+		if len(table.PrimaryKeys) != 1 {
+			return fmt.Errorf("table %s has %d primary keys, only the table with exactly one primary key can be iterated", table.Name, len(table.PrimaryKeys))
+		}
+		colName = table.PrimaryKeys[0]
+	}
+
+	col = table.GetColumn(colName)
+	batchSize := setting.Database.IterateBufferSize
+	var lastColValue any
+	for {
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+
+		beans := make([]*Bean, 0, batchSize)
+		query := GetEngine(ctx).Table(table.Name).Asc(colName)
+
+		batchCond := cond
+		if lastColValue != nil {
+			batchCond = builder.And(cond, builder.Gt{col.Name: lastColValue})
+		}
+		if batchCond != nil {
+			query = query.Where(batchCond)
+		}
+
+		if err := query.Limit(batchSize).Find(&beans); err != nil {
+			return err
+		}
+		if len(beans) == 0 {
+			return nil
+		}
+
+		reflectVal, err := col.ValueOf(beans[len(beans)-1])
+		if err != nil {
+			return err
+		}
+		lastColValue = reflectVal.Interface()
+
+		for _, bean := range beans {
+			if err := f(ctx, bean); err != nil {
+				return err
 			}
 		}
 	}
 }
+
+func IterateByColumn[Bean any](ctx context.Context, colName string, cond builder.Cond, f func(ctx context.Context, bean *Bean) error) error {
+	return iterateTableByColumn(ctx, colName, cond, f)
+}
+
+func Iterate[Bean any](ctx context.Context, cond builder.Cond, f func(ctx context.Context, bean *Bean) error) error {
+	return iterateTableByColumn(ctx, "", cond, f)
+}

models/db/list.go

@@ -38,10 +38,7 @@ type ListOptions struct {
 
 var ListOptionsAll = ListOptions{ListAll: true}
 
-var (
-	_ Paginator   = &ListOptions{}
-	_ FindOptions = ListOptions{}
-)
+var _ Paginator = &ListOptions{}
 
 // GetSkipTake returns the skip and take values
 func (opts *ListOptions) GetSkipTake() (skip, take int) {
@@ -117,6 +114,7 @@ type FindOptions interface {
 	GetPageSize() int
 	IsListAll() bool
 	ToConds() builder.Cond
+	ToOrders() string
 }
 
 type JoinFunc func(sess Engine) error
@@ -125,10 +123,6 @@ type FindOptionsJoin interface {
 	ToJoins() []JoinFunc
 }
 
-type FindOptionsOrder interface {
-	ToOrders() string
-}
-
 // Find represents a common find function which accept an options interface
 func Find[T any](ctx context.Context, opts FindOptions) ([]*T, error) {
 	sess := GetEngine(ctx).Where(opts.ToConds())
@@ -140,12 +134,7 @@ func Find[T any](ctx context.Context, opts FindOptions) ([]*T, error) {
 			}
 		}
 	}
-	if orderOpt, ok := opts.(FindOptionsOrder); ok {
-		if order := orderOpt.ToOrders(); order != "" {
-			sess.OrderBy(order)
-		}
-	}
-
+	sess.OrderBy(opts.ToOrders())
 	page, pageSize := opts.GetPage(), opts.GetPageSize()
 	if !opts.IsListAll() && pageSize > 0 {
 		if page == 0 {
@@ -167,15 +156,17 @@ func Find[T any](ctx context.Context, opts FindOptions) ([]*T, error) {
 
 // Count represents a common count function which accept an options interface
 func Count[T any](ctx context.Context, opts FindOptions) (int64, error) {
-	sess := GetEngine(ctx).Where(opts.ToConds())
-	if joinOpt, ok := opts.(FindOptionsJoin); ok {
-		for _, joinFunc := range joinOpt.ToJoins() {
-			if err := joinFunc(sess); err != nil {
-				return 0, err
+	sess := GetEngine(ctx)
+	if opts != nil {
+		sess.Where(opts.ToConds())
+		if joinOpt, ok := opts.(FindOptionsJoin); ok {
+			for _, joinFunc := range joinOpt.ToJoins() {
+				if err := joinFunc(sess); err != nil {
+					return 0, err
+				}
 			}
 		}
 	}
-
 	var object T
 	return sess.Count(&object)
 }
@@ -194,11 +185,7 @@ func FindAndCount[T any](ctx context.Context, opts FindOptions) ([]*T, int64, er
 			}
 		}
 	}
-	if orderOpt, ok := opts.(FindOptionsOrder); ok {
-		if order := orderOpt.ToOrders(); order != "" {
-			sess.OrderBy(order)
-		}
-	}
+	sess.OrderBy(opts.ToOrders())
 
 	findPageSize := defaultFindSliceSize
 	if pageSize > 0 {

models/db/list_test.go

@@ -18,6 +18,10 @@ type mockListOptions struct {
 	db.ListOptions
 }
 
+func (opts mockListOptions) ToOrders() string {
+	return "id"
+}
+
 func (opts mockListOptions) IsListAll() bool {
 	return true
 }