fix(indexer): correct bleve indexer token filters (#38853)

* fix #36228
* fix #37221

---------

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

modules/indexer/code/bleve/bleve.go

@@ -32,7 +32,6 @@ import (
 	analyzer_keyword "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
 	"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
 	"github.com/blevesearch/bleve/v2/analysis/token/unicodenorm"
-	"github.com/blevesearch/bleve/v2/analysis/tokenizer/letter"
 	"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
 	"github.com/blevesearch/bleve/v2/mapping"
 	"github.com/blevesearch/bleve/v2/search/query"
@@ -69,9 +68,8 @@ func (d *RepoIndexerData) Type() string {
 const (
 	repoIndexerAnalyzer      = "repoIndexerAnalyzer"
 	filenameIndexerAnalyzer  = "filenameIndexerAnalyzer"
-	filenameIndexerTokenizer = "filenameIndexerTokenizer"
 	repoIndexerDocType       = "repoIndexerDocType"
-	repoIndexerLatestVersion = 9
+	repoIndexerLatestVersion = 10
 )
 
 // generateBleveIndexMapping generates a bleve index mapping for the repo indexer
@@ -107,8 +105,8 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) {
 	} else if err := mapping.AddCustomAnalyzer(repoIndexerAnalyzer, map[string]any{
 		"type":          analyzer_custom.Name,
 		"char_filters":  []string{},
-		"tokenizer":     letter.Name,
-		"token_filters": []string{unicodeNormalizeName, lowercase.Name},
+		"tokenizer":     codeTokenizerName,
+		"token_filters": []string{unicodeNormalizeName, codeTokenFilterName, lowercase.Name},
 	}); err != nil {
 		return nil, err
 	}

modules/indexer/code/bleve/bleve_test.go

@@ -0,0 +1,59 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package bleve
+
+import (
+	"testing"
+	"time"
+
+	"gitea.dev/models/db"
+	"gitea.dev/modules/indexer/code/internal"
+	inner_bleve "gitea.dev/modules/indexer/internal/bleve"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBleveIndexerTokenFilter(t *testing.T) {
+	dir := t.TempDir()
+	indexer := NewIndexer(dir)
+	defer indexer.Close()
+
+	_, err := indexer.Init(t.Context())
+	require.NoError(t, err)
+
+	batch := inner_bleve.NewFlushingBatch(indexer.inner.Indexer, maxBatchSize)
+	batch.Index("2", &RepoIndexerData{RepoID: 2, Content: "mDNS.port2=12345", UpdatedAt: time.Now()})
+	batch.Flush()
+
+	testCases := []struct {
+		keyword     string
+		expectedIDs []int64
+	}{
+		{keyword: "12345", expectedIDs: []int64{2}},
+		{keyword: "DNS", expectedIDs: []int64{}},
+		{keyword: "mdns", expectedIDs: []int64{2}},
+		{keyword: "port", expectedIDs: []int64{2}},
+		{keyword: "port2", expectedIDs: []int64{2}},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.keyword, func(t *testing.T) {
+			_, results, _, err := indexer.Search(t.Context(), &internal.SearchOptions{
+				Paginator: &db.ListOptions{Page: 1, PageSize: 1},
+				Keyword:   testCase.keyword,
+			})
+			require.NoError(t, err)
+			assert.ElementsMatch(t, testCase.expectedIDs, searchResultIDs(results))
+		})
+	}
+}
+
+func searchResultIDs(result []*internal.SearchResult) []int64 {
+	ids := make([]int64, 0, len(result))
+	for _, hit := range result {
+		ids = append(ids, hit.RepoID)
+	}
+	return ids
+}

modules/indexer/code/bleve/code_token.go

@@ -0,0 +1,65 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package bleve
+
+import (
+	"regexp"
+	"unicode"
+
+	"gitea.dev/modules/util"
+
+	"github.com/blevesearch/bleve/v2/analysis"
+	"github.com/blevesearch/bleve/v2/analysis/tokenizer/character"
+	"github.com/blevesearch/bleve/v2/registry"
+)
+
+const codeTokenizerName = "codeTokenizer"
+
+func codeTokenizerConstructor(_ map[string]any, _ *registry.Cache) (analysis.Tokenizer, error) {
+	// Old code used "letter" tokenizer which doesn't support CJK.
+	// Here it still doesn't support CJK, since there is no usable CJK tokenizer at the moment.
+	return character.NewCharacterTokenizer(func(r rune) bool {
+		return unicode.IsLetter(r) || unicode.IsNumber(r)
+	}), nil
+}
+
+const codeTokenFilterName = "codeTokenFilter"
+
+type codeTokenFilter struct {
+	re *regexp.Regexp
+}
+
+func (c codeTokenFilter) Filter(stream analysis.TokenStream) (ret analysis.TokenStream) {
+	// split one token to "letter" parts and "number" parts (to keep the old behavior).
+	// e.g.: input token="port123", then the output tokens are "port123", "port", "123"
+	for _, token := range stream {
+		ret = append(ret, token)
+		m := c.re.FindAllIndex(token.Term, -1)
+		if len(m) > 1 {
+			for _, it := range m {
+				p1, p2 := it[0], it[1]
+				t := &analysis.Token{
+					Start:    token.Start + p1,
+					End:      token.Start + p2,
+					Term:     token.Term[p1:p2],
+					Position: token.Position,
+					Type:     analysis.AlphaNumeric,
+				}
+				ret = append(ret, t)
+			}
+		}
+	}
+	return ret
+}
+
+func codeTokenFilterConstructor(_ map[string]any, _ *registry.Cache) (analysis.TokenFilter, error) {
+	return &codeTokenFilter{
+		re: regexp.MustCompile("[a-zA-Z]+|[0-9]+"),
+	}, nil
+}
+
+func init() {
+	util.MustNoError(registry.RegisterTokenizer(codeTokenizerName, codeTokenizerConstructor))
+	util.MustNoError(registry.RegisterTokenFilter(codeTokenFilterName, codeTokenFilterConstructor))
+}

modules/indexer/code/bleve/token/path/path.go

@@ -7,13 +7,13 @@ import (
 	"slices"
 	"strings"
 
+	"gitea.dev/modules/util"
+
 	"github.com/blevesearch/bleve/v2/analysis"
 	"github.com/blevesearch/bleve/v2/registry"
 )
 
-const (
-	Name = "gitea/path"
-)
+const Name = "gitea/path"
 
 type TokenFilter struct{}
 
@@ -98,8 +98,5 @@ func generatePathTokens(input analysis.TokenStream, reversed bool) analysis.Toke
 
 func init() {
 	// FIXME: move it to the bleve's init function, but do not call it in global init
-	err := registry.RegisterTokenFilter(Name, TokenFilterConstructor)
-	if err != nil {
-		panic(err)
-	}
+	util.MustNoError(registry.RegisterTokenFilter(Name, TokenFilterConstructor))
 }