refactor: only reset a database table when the table's data was changed (#37573)

Reduce CI time

Saves about 3 minutes for each test suit

test-unit: 13min -> 10min (-race)
test-pgsql: 24min -> 20min (-race)
test-mysql: 15min -> 12min
test-mssql: 16min -> 12min

---------

Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: Claude (Opus 4.7) <noreply@anthropic.com>

models/unittest/fixtures.go

@@ -4,7 +4,10 @@
 package unittest
 
 import (
+	"context"
 	"fmt"
+	"strings"
+	"unicode"
 
 	"code.gitea.io/gitea/models/db"
 	"code.gitea.io/gitea/modules/auth/password/hash"
@@ -12,11 +15,13 @@ import (
 	"code.gitea.io/gitea/modules/util"
 
 	"xorm.io/xorm"
+	"xorm.io/xorm/contexts"
 	"xorm.io/xorm/schemas"
 )
 
 type FixturesLoader interface {
 	Load() error
+	MarkTableChanged(tableName string)
 }
 
 var fixturesLoader FixturesLoader
@@ -57,15 +62,101 @@ func loadFixtureResetSeqPgsql(e *xorm.Engine) error {
 	return nil
 }
 
+type fixturesHookStruct struct{}
+
+func cutSpaceForSQL(s string) (string, string, bool) {
+	s = strings.TrimSpace(s)
+	pos := strings.IndexFunc(s, unicode.IsSpace)
+	if pos == -1 {
+		return s, "", false
+	}
+	return s[:pos], strings.TrimSpace(s[pos+1:]), true
+}
+
+func trimTableNameQuotes(s string) string {
+	pos := strings.IndexByte(s, '.')
+	if pos != -1 {
+		s = s[pos+1:]
+	}
+	return strings.Trim(s, "\"`[]")
+}
+
+func (f fixturesHookStruct) BeforeProcess(c *contexts.ContextHook) (context.Context, error) {
+	if c.Ctx.Value(db.ContextKeyTestFixtures) != nil {
+		return c.Ctx, nil
+	}
+	ctx, sql := c.Ctx, c.SQL
+	cmdPart, cmdRemaining, ok := cutSpaceForSQL(sql)
+	if !ok {
+		return ctx, nil
+	}
+
+	// ignore the SQLs which don't change data
+	if util.AsciiEqualFold(cmdPart, "SELECT") ||
+		util.AsciiEqualFold(cmdPart, "SHOW") ||
+		util.AsciiEqualFold(cmdPart, "PRAGMA") ||
+		util.AsciiEqualFold(cmdPart, "ALTER") ||
+		util.AsciiEqualFold(cmdPart, "CREATE") ||
+		util.AsciiEqualFold(cmdPart, "DROP") ||
+		util.AsciiEqualFold(cmdPart, "IF") ||
+		util.AsciiEqualFold(cmdPart, "SET") ||
+		util.AsciiEqualFold(cmdPart, "sp_rename") ||
+		util.AsciiEqualFold(cmdPart, "BEGIN") ||
+		util.AsciiEqualFold(cmdPart, "ROLLBACK") ||
+		util.AsciiEqualFold(cmdPart, "COMMIT") {
+		return ctx, nil
+	}
+
+	switch {
+	case util.AsciiEqualFold(cmdPart, "INSERT"):
+		cmdPart, cmdRemaining, _ = cutSpaceForSQL(cmdRemaining)
+		if util.AsciiEqualFold(cmdPart, "INTO") {
+			cmdPart, cmdRemaining, _ = cutSpaceForSQL(cmdRemaining)
+		}
+		fixturesLoader.MarkTableChanged(trimTableNameQuotes(cmdPart))
+	case util.AsciiEqualFold(cmdPart, "MERGE"):
+		cmdPart, cmdRemaining, _ = cutSpaceForSQL(cmdRemaining)
+		if util.AsciiEqualFold(cmdPart, "INTO") {
+			cmdPart, cmdRemaining, _ = cutSpaceForSQL(cmdRemaining)
+		}
+		fixturesLoader.MarkTableChanged(trimTableNameQuotes(cmdPart))
+	case util.AsciiEqualFold(cmdPart, "UPDATE"):
+		cmdPart, cmdRemaining, _ = cutSpaceForSQL(cmdRemaining)
+		fixturesLoader.MarkTableChanged(trimTableNameQuotes(cmdPart))
+	case util.AsciiEqualFold(cmdPart, "DELETE"):
+		cmdPart, cmdRemaining, _ = cutSpaceForSQL(cmdRemaining)
+		if util.AsciiEqualFold(cmdPart, "FROM") {
+			cmdPart, cmdRemaining, _ = cutSpaceForSQL(cmdRemaining)
+		}
+		fixturesLoader.MarkTableChanged(trimTableNameQuotes(cmdPart))
+	case util.AsciiEqualFold(cmdPart, "TRUNCATE"):
+		cmdPart, cmdRemaining, _ = cutSpaceForSQL(cmdRemaining)
+		if util.AsciiEqualFold(cmdPart, "TABLE") {
+			cmdPart, cmdRemaining, _ = cutSpaceForSQL(cmdRemaining)
+		}
+		fixturesLoader.MarkTableChanged(trimTableNameQuotes(cmdPart))
+	default:
+		// should either parse the table name if it changes data, or ignore it
+		panic("unrecognized sql: " + sql)
+	}
+	_ = cmdRemaining
+	return ctx, nil
+}
+
+func (f fixturesHookStruct) AfterProcess(c *contexts.ContextHook) error {
+	return nil
+}
+
 // InitFixtures initialize test fixtures for a test database
-func InitFixtures(opts FixturesOptions, engine ...*xorm.Engine) (err error) {
-	xormEngine := util.IfZero(util.OptionalArg(engine), GetXORMEngine())
+func InitFixtures(opts FixturesOptions) (err error) {
+	xormEngine := GetXORMEngine()
 	fixturesLoader, err = NewFixturesLoader(xormEngine, opts)
 	// fixturesLoader = NewFixturesLoaderVendor(xormEngine, opts)
 
 	// register the dummy hash algorithm function used in the test fixtures
 	_ = hash.Register("dummy", hash.NewDummyHasher)
 	setting.PasswordHashAlgo, _ = hash.SetDefaultPasswordHashAlgorithm("dummy")
+	xormEngine.AddHook(&fixturesHookStruct{})
 	return err
 }
 

models/unittest/fixtures_loader.go

@@ -4,6 +4,7 @@
 package unittest
 
 import (
+	"context"
 	"database/sql"
 	"encoding/hex"
 	"fmt"
@@ -11,6 +12,7 @@ import (
 	"path/filepath"
 	"slices"
 	"strings"
+	"sync"
 
 	"code.gitea.io/gitea/models/db"
 
@@ -32,7 +34,7 @@ type FixtureItem struct {
 
 type fixturesLoaderInternal struct {
 	xormEngine       *xorm.Engine
-	xormTableNames   map[string]bool
+	tableSyncMap     sync.Map
 	db               *sql.DB
 	dbType           schemas.DBType
 	fixtures         map[string]*FixtureItem
@@ -148,25 +150,36 @@ func (f *fixturesLoaderInternal) Load() error {
 	}
 	defer func() { _ = tx.Rollback() }()
 
+	ctx := context.WithValue(context.Background(), db.ContextKeyTestFixtures, true)
+
 	for _, fixture := range f.fixtures {
-		if !f.xormTableNames[fixture.tableName] {
+		synced, existing := f.tableSyncMap.Load(fixture.tableName)
+		if synced == true || !existing {
 			continue
 		}
 		if err := f.loadFixtures(tx, fixture); err != nil {
 			return fmt.Errorf("failed to load fixtures from %s: %w", fixture.fileFullPath, err)
 		}
+		f.tableSyncMap.Store(fixture.tableName, true)
 	}
 	if err = tx.Commit(); err != nil {
 		return err
 	}
-	for xormTableName := range f.xormTableNames {
-		if f.fixtures[xormTableName] == nil {
-			_, _ = f.xormEngine.Exec("DELETE FROM `" + xormTableName + "`")
+	f.tableSyncMap.Range(func(k, v any) bool {
+		tableName, synced := k.(string), v.(bool)
+		if !synced && f.fixtures[tableName] == nil {
+			_, _ = f.xormEngine.Context(ctx).Exec("DELETE FROM `" + tableName + "`")
 		}
-	}
+		f.tableSyncMap.Store(tableName, true)
+		return true
+	})
 	return nil
 }
 
+func (f *fixturesLoaderInternal) MarkTableChanged(tableName string) {
+	f.tableSyncMap.Store(tableName, false)
+}
+
 func FixturesFileFullPaths(dir string, files []string) (map[string]*FixtureItem, error) {
 	if files != nil && len(files) == 0 {
 		return nil, nil //nolint:nilnil // load nothing
@@ -215,11 +228,12 @@ func NewFixturesLoader(x *xorm.Engine, opts FixturesOptions) (FixturesLoader, er
 		f.paramPlaceholder = func(idx int) string { return "?" }
 	}
 
+	// If a model is not imported in a package (no bean is registered), the table won't exist in database.
+	// So only use tables of registered models (beans).
 	xormBeans, _ := db.NamesToBean()
-	f.xormTableNames = map[string]bool{}
 	for _, bean := range xormBeans {
-		f.xormTableNames[x.TableName(bean)] = true
+		beanTableName := x.TableName(bean)
+		f.tableSyncMap.Store(trimTableNameQuotes(beanTableName), false)
 	}
-
 	return f, nil
 }