fix(indexer): correct bleve indexer token filters (#38853)

* fix #36228
* fix #37221

---------

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

modules/indexer/code/bleve/bleve.go

@@ -32,7 +32,6 @@ import (
 	analyzer_keyword "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
 	"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
 	"github.com/blevesearch/bleve/v2/analysis/token/unicodenorm"
-	"github.com/blevesearch/bleve/v2/analysis/tokenizer/letter"
 	"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
 	"github.com/blevesearch/bleve/v2/mapping"
 	"github.com/blevesearch/bleve/v2/search/query"
@@ -69,9 +68,8 @@ func (d *RepoIndexerData) Type() string {
 const (
 	repoIndexerAnalyzer      = "repoIndexerAnalyzer"
 	filenameIndexerAnalyzer  = "filenameIndexerAnalyzer"
-	filenameIndexerTokenizer = "filenameIndexerTokenizer"
 	repoIndexerDocType       = "repoIndexerDocType"
-	repoIndexerLatestVersion = 9
+	repoIndexerLatestVersion = 10
 )
 
 // generateBleveIndexMapping generates a bleve index mapping for the repo indexer
@@ -107,8 +105,8 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) {
 	} else if err := mapping.AddCustomAnalyzer(repoIndexerAnalyzer, map[string]any{
 		"type":          analyzer_custom.Name,
 		"char_filters":  []string{},
-		"tokenizer":     letter.Name,
+		"tokenizer":     codeTokenizerName,
-		"token_filters": []string{unicodeNormalizeName, lowercase.Name},
+		"token_filters": []string{unicodeNormalizeName, codeTokenFilterName, lowercase.Name},
 	}); err != nil {
 		return nil, err
 	}

modules/indexer/code/bleve/bleve_test.go

@@ -0,0 +1,59 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package bleve
+
+import (
+	"testing"
+	"time"
+
+	"gitea.dev/models/db"
+	"gitea.dev/modules/indexer/code/internal"
+	inner_bleve "gitea.dev/modules/indexer/internal/bleve"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBleveIndexerTokenFilter(t *testing.T) {
+	dir := t.TempDir()
+	indexer := NewIndexer(dir)
+	defer indexer.Close()
+
+	_, err := indexer.Init(t.Context())
+	require.NoError(t, err)
+
+	batch := inner_bleve.NewFlushingBatch(indexer.inner.Indexer, maxBatchSize)
+	batch.Index("2", &RepoIndexerData{RepoID: 2, Content: "mDNS.port2=12345", UpdatedAt: time.Now()})
+	batch.Flush()
+
+	testCases := []struct {
+		keyword     string
+		expectedIDs []int64
+	}{
+		{keyword: "12345", expectedIDs: []int64{2}},
+		{keyword: "DNS", expectedIDs: []int64{}},
+		{keyword: "mdns", expectedIDs: []int64{2}},
+		{keyword: "port", expectedIDs: []int64{2}},
+		{keyword: "port2", expectedIDs: []int64{2}},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.keyword, func(t *testing.T) {
+			_, results, _, err := indexer.Search(t.Context(), &internal.SearchOptions{
+				Paginator: &db.ListOptions{Page: 1, PageSize: 1},
+				Keyword:   testCase.keyword,
+			})
+			require.NoError(t, err)
+			assert.ElementsMatch(t, testCase.expectedIDs, searchResultIDs(results))
+		})
+	}
+}
+
+func searchResultIDs(result []*internal.SearchResult) []int64 {
+	ids := make([]int64, 0, len(result))
+	for _, hit := range result {
+		ids = append(ids, hit.RepoID)
+	}
+	return ids
+}

modules/indexer/code/bleve/code_token.go

@@ -0,0 +1,65 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package bleve
+
+import (
+	"regexp"
+	"unicode"
+
+	"gitea.dev/modules/util"
+
+	"github.com/blevesearch/bleve/v2/analysis"
+	"github.com/blevesearch/bleve/v2/analysis/tokenizer/character"
+	"github.com/blevesearch/bleve/v2/registry"
+)
+
+const codeTokenizerName = "codeTokenizer"
+
+func codeTokenizerConstructor(_ map[string]any, _ *registry.Cache) (analysis.Tokenizer, error) {
+	// Old code used "letter" tokenizer which doesn't support CJK.
+	// Here it still doesn't support CJK, since there is no usable CJK tokenizer at the moment.
+	return character.NewCharacterTokenizer(func(r rune) bool {
+		return unicode.IsLetter(r) || unicode.IsNumber(r)
+	}), nil
+}
+
+const codeTokenFilterName = "codeTokenFilter"
+
+type codeTokenFilter struct {
+	re *regexp.Regexp
+}
+
+func (c codeTokenFilter) Filter(stream analysis.TokenStream) (ret analysis.TokenStream) {
+	// split one token to "letter" parts and "number" parts (to keep the old behavior).
+	// e.g.: input token="port123", then the output tokens are "port123", "port", "123"
+	for _, token := range stream {
+		ret = append(ret, token)
+		m := c.re.FindAllIndex(token.Term, -1)
+		if len(m) > 1 {
+			for _, it := range m {
+				p1, p2 := it[0], it[1]
+				t := &analysis.Token{
+					Start:    token.Start + p1,
+					End:      token.Start + p2,
+					Term:     token.Term[p1:p2],
+					Position: token.Position,
+					Type:     analysis.AlphaNumeric,
+				}
+				ret = append(ret, t)
+			}
+		}
+	}
+	return ret
+}
+
+func codeTokenFilterConstructor(_ map[string]any, _ *registry.Cache) (analysis.TokenFilter, error) {
+	return &codeTokenFilter{
+		re: regexp.MustCompile("[a-zA-Z]+|[0-9]+"),
+	}, nil
+}
+
+func init() {
+	util.MustNoError(registry.RegisterTokenizer(codeTokenizerName, codeTokenizerConstructor))
+	util.MustNoError(registry.RegisterTokenFilter(codeTokenFilterName, codeTokenFilterConstructor))
+}

modules/indexer/code/bleve/token/path/path.go

@@ -7,13 +7,13 @@ import (
 	"slices"
 	"strings"
 
+	"gitea.dev/modules/util"
+
 	"github.com/blevesearch/bleve/v2/analysis"
 	"github.com/blevesearch/bleve/v2/registry"
 )
 
-const (
+const Name = "gitea/path"
-	Name = "gitea/path"
-)
 
 type TokenFilter struct{}
 
@@ -98,8 +98,5 @@ func generatePathTokens(input analysis.TokenStream, reversed bool) analysis.Toke
 
 func init() {
 	// FIXME: move it to the bleve's init function, but do not call it in global init
-	err := registry.RegisterTokenFilter(Name, TokenFilterConstructor)
+	util.MustNoError(registry.RegisterTokenFilter(Name, TokenFilterConstructor))
-	if err != nil {
-		panic(err)
-	}
 }

modules/indexer/issues/bleve/bleve.go

@@ -15,7 +15,6 @@ import (
 
 	"github.com/blevesearch/bleve/v2"
 	"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
-	"github.com/blevesearch/bleve/v2/analysis/token/camelcase"
 	"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
 	"github.com/blevesearch/bleve/v2/analysis/token/unicodenorm"
 	"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
@@ -26,7 +25,7 @@ import (
 const (
 	issueIndexerAnalyzer      = "issueIndexer"
 	issueIndexerDocType       = "issueIndexerDocType"
-	issueIndexerLatestVersion = 7
+	issueIndexerLatestVersion = 8
 )
 
 const unicodeNormalizeName = "unicodeNormalize"
@@ -103,7 +102,7 @@ func generateIssueIndexMapping() (mapping.IndexMapping, error) {
 		"type":          custom.Name,
 		"char_filters":  []string{},
 		"tokenizer":     unicode.Name,
-		"token_filters": []string{unicodeNormalizeName, camelcase.Name, lowercase.Name},
+		"token_filters": []string{unicodeNormalizeName, camelCaseKeepWholeName, lowercase.Name},
 	}); err != nil {
 		return nil, err
 	}

modules/indexer/issues/bleve/bleve_test.go

@@ -77,6 +77,47 @@ func TestBleveIndexerNoAssignee(t *testing.T) {
 	}
 }
 
+func TestBleveIndexerTokenFilter(t *testing.T) {
+	dir := t.TempDir()
+	indexer := NewIndexer(dir)
+	defer indexer.Close()
+
+	_, err := indexer.Init(t.Context())
+	require.NoError(t, err)
+
+	require.NoError(t, indexer.Index(t.Context(),
+		&internal.IndexerData{ID: 1, Title: "fix(packages): SomeThing needs a rewrite (#12345)"},
+		&internal.IndexerData{ID: 2, Title: "add support for mDNS discovery abc1234"},
+	))
+
+	testCases := []struct {
+		name        string
+		keyword     string
+		expectedIDs []int64
+	}{
+		{name: "exact original case", keyword: "SomeThing", expectedIDs: []int64{1}},
+		{name: "case matching original transitions", keyword: "someThing", expectedIDs: []int64{1}},
+		{name: "all lower case", keyword: "something", expectedIDs: []int64{1}},
+		{name: "all upper case", keyword: "SOMETHING", expectedIDs: []int64{1}},
+		{name: "number match", keyword: "12345", expectedIDs: []int64{1}},
+		{name: "number as part", keyword: "1234", expectedIDs: []int64{2}},
+		{name: "sub-word search still works", keyword: "DNS", expectedIDs: []int64{2}},
+		{name: "sub-word search, lower case", keyword: "mdns", expectedIDs: []int64{2}},
+		{name: "keyword is camel case", keyword: "addSupport", expectedIDs: []int64{2}},
+		{name: "keyword not match", keyword: "addsupport", expectedIDs: []int64{}},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			result, err := indexer.Search(t.Context(), &internal.SearchOptions{
+				Keyword: testCase.keyword,
+			})
+			require.NoError(t, err)
+			assert.ElementsMatch(t, testCase.expectedIDs, searchResultIDs(result))
+		})
+	}
+}
+
 func searchResultIDs(result *internal.SearchResult) []int64 {
 	ids := make([]int64, 0, len(result.Hits))
 	for _, hit := range result.Hits {

modules/indexer/issues/bleve/camelcase_keep_whole.go

@@ -0,0 +1,64 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package bleve
+
+import (
+	"gitea.dev/modules/util"
+
+	"github.com/blevesearch/bleve/v2/analysis"
+	"github.com/blevesearch/bleve/v2/analysis/token/camelcase"
+	"github.com/blevesearch/bleve/v2/registry"
+)
+
+const camelCaseKeepWholeName = "camelCaseKeepWhole"
+
+// camelCaseKeepWholeFilter behaves like bleve's built-in "camelCase" token filter,
+// it also uses the whole word for a token. For example: when indexing "someThing",
+// CamelCaseFilter only emits "some" and "thing", this filter also emits "something".
+// It is questionable why the "issue indexer" used the CamelCaseFilter, it just can't search "someThing".
+// To avoid breaking existing user experiences, this "whole token filter" is introduced to make the full word can be searched.
+type camelCaseKeepWholeFilter struct {
+	inner *camelcase.CamelCaseFilter
+}
+
+func (f *camelCaseKeepWholeFilter) Filter(input analysis.TokenStream) analysis.TokenStream {
+	// First, do exactly what the stock camelCase filter does: split by "camelCase" tokens
+	split := f.inner.Filter(input)
+
+	// Index the resulting position of the *first* sub-token produced for
+	// each original token (matched by start offset), so the duplicated
+	// whole-word token we add below lines up at the same position as the
+	// sub-word it stands in for, instead of drifting out of sync for
+	// fields with more than one original token.
+	posByStart := make(map[int]int, len(split))
+	for _, tok := range split {
+		if _, ok := posByStart[tok.Start]; !ok {
+			posByStart[tok.Start] = tok.Position
+		}
+	}
+
+	rv := make(analysis.TokenStream, 0, len(split)+len(input))
+	rv = append(rv, split...)
+
+	// Then append one extra, un-split copy of every original token, so the
+	// whole word survives as a standalone, independently searchable term.
+	for _, token := range input {
+		dup := *token
+		dup.Term = append([]byte(nil), token.Term...)
+		if pos, ok := posByStart[token.Start]; ok {
+			dup.Position = pos
+		}
+		rv = append(rv, &dup)
+	}
+
+	return rv
+}
+
+func camelCaseKeepWholeFilterConstructor(_ map[string]any, _ *registry.Cache) (analysis.TokenFilter, error) {
+	return &camelCaseKeepWholeFilter{inner: camelcase.NewCamelCaseFilter()}, nil
+}
+
+func init() {
+	util.MustNoError(registry.RegisterTokenFilter(camelCaseKeepWholeName, camelCaseKeepWholeFilterConstructor))
+}

modules/util/util.go

@@ -317,3 +317,9 @@ func DiffSlice[T comparable](oldSlice, newSlice []T) (added, removed []T) {
 	}
 	return added, removed
 }
+
+func MustNoError(err error) {
+	if err != nil {
+		panic(err)
+	}
+}

tests/integration/repo_search_test.go

@@ -53,9 +53,9 @@ func TestSearchRepo(t *testing.T) {
 }
 
 func testSearch(t *testing.T, url string, expected []string) {
+	t.Helper()
 	req := NewRequest(t, "GET", url)
 	resp := MakeRequest(t, req, http.StatusOK)
-
 	filenames := resultFilenames(NewHTMLParser(t, resp.Body))
-	assert.Equal(t, expected, filenames)
+	assert.Equal(t, expected, filenames, "url=%s", url)
 }