From 1b21c8a1e6e56cdafb87abd1a1322217dbdbd9e2 Mon Sep 17 00:00:00 2001 From: Mitrahsoft Date: Mon, 17 Aug 2026 13:29:41 +0530 Subject: [PATCH] fix(indexer): correct bleve indexer token filters (#38853) * fix #36228 * fix #37221 --------- Co-authored-by: wxiaoguang --- modules/indexer/code/bleve/bleve.go | 8 +-- modules/indexer/code/bleve/bleve_test.go | 59 +++++++++++++++++ modules/indexer/code/bleve/code_token.go | 65 +++++++++++++++++++ modules/indexer/code/bleve/token/path/path.go | 11 ++-- modules/indexer/issues/bleve/bleve.go | 5 +- modules/indexer/issues/bleve/bleve_test.go | 41 ++++++++++++ .../issues/bleve/camelcase_keep_whole.go | 64 ++++++++++++++++++ modules/util/util.go | 6 ++ tests/integration/repo_search_test.go | 4 +- 9 files changed, 246 insertions(+), 17 deletions(-) create mode 100644 modules/indexer/code/bleve/bleve_test.go create mode 100644 modules/indexer/code/bleve/code_token.go create mode 100644 modules/indexer/issues/bleve/camelcase_keep_whole.go diff --git a/modules/indexer/code/bleve/bleve.go b/modules/indexer/code/bleve/bleve.go index 6e78e42efb1..46f97624a8c 100644 --- a/modules/indexer/code/bleve/bleve.go +++ b/modules/indexer/code/bleve/bleve.go @@ -32,7 +32,6 @@ import ( analyzer_keyword "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" "github.com/blevesearch/bleve/v2/analysis/token/lowercase" "github.com/blevesearch/bleve/v2/analysis/token/unicodenorm" - "github.com/blevesearch/bleve/v2/analysis/tokenizer/letter" "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search/query" @@ -69,9 +68,8 @@ func (d *RepoIndexerData) Type() string { const ( repoIndexerAnalyzer = "repoIndexerAnalyzer" filenameIndexerAnalyzer = "filenameIndexerAnalyzer" - filenameIndexerTokenizer = "filenameIndexerTokenizer" repoIndexerDocType = "repoIndexerDocType" - repoIndexerLatestVersion = 9 + repoIndexerLatestVersion = 10 ) // generateBleveIndexMapping generates a bleve index mapping for the repo indexer @@ -107,8 +105,8 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) { } else if err := mapping.AddCustomAnalyzer(repoIndexerAnalyzer, map[string]any{ "type": analyzer_custom.Name, "char_filters": []string{}, - "tokenizer": letter.Name, - "token_filters": []string{unicodeNormalizeName, lowercase.Name}, + "tokenizer": codeTokenizerName, + "token_filters": []string{unicodeNormalizeName, codeTokenFilterName, lowercase.Name}, }); err != nil { return nil, err } diff --git a/modules/indexer/code/bleve/bleve_test.go b/modules/indexer/code/bleve/bleve_test.go new file mode 100644 index 00000000000..0d0fac0f312 --- /dev/null +++ b/modules/indexer/code/bleve/bleve_test.go @@ -0,0 +1,59 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package bleve + +import ( + "testing" + "time" + + "gitea.dev/models/db" + "gitea.dev/modules/indexer/code/internal" + inner_bleve "gitea.dev/modules/indexer/internal/bleve" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBleveIndexerTokenFilter(t *testing.T) { + dir := t.TempDir() + indexer := NewIndexer(dir) + defer indexer.Close() + + _, err := indexer.Init(t.Context()) + require.NoError(t, err) + + batch := inner_bleve.NewFlushingBatch(indexer.inner.Indexer, maxBatchSize) + batch.Index("2", &RepoIndexerData{RepoID: 2, Content: "mDNS.port2=12345", UpdatedAt: time.Now()}) + batch.Flush() + + testCases := []struct { + keyword string + expectedIDs []int64 + }{ + {keyword: "12345", expectedIDs: []int64{2}}, + {keyword: "DNS", expectedIDs: []int64{}}, + {keyword: "mdns", expectedIDs: []int64{2}}, + {keyword: "port", expectedIDs: []int64{2}}, + {keyword: "port2", expectedIDs: []int64{2}}, + } + + for _, testCase := range testCases { + t.Run(testCase.keyword, func(t *testing.T) { + _, results, _, err := indexer.Search(t.Context(), &internal.SearchOptions{ + Paginator: &db.ListOptions{Page: 1, PageSize: 1}, + Keyword: testCase.keyword, + }) + require.NoError(t, err) + assert.ElementsMatch(t, testCase.expectedIDs, searchResultIDs(results)) + }) + } +} + +func searchResultIDs(result []*internal.SearchResult) []int64 { + ids := make([]int64, 0, len(result)) + for _, hit := range result { + ids = append(ids, hit.RepoID) + } + return ids +} diff --git a/modules/indexer/code/bleve/code_token.go b/modules/indexer/code/bleve/code_token.go new file mode 100644 index 00000000000..3a14ddf3981 --- /dev/null +++ b/modules/indexer/code/bleve/code_token.go @@ -0,0 +1,65 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package bleve + +import ( + "regexp" + "unicode" + + "gitea.dev/modules/util" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/character" + "github.com/blevesearch/bleve/v2/registry" +) + +const codeTokenizerName = "codeTokenizer" + +func codeTokenizerConstructor(_ map[string]any, _ *registry.Cache) (analysis.Tokenizer, error) { + // Old code used "letter" tokenizer which doesn't support CJK. + // Here it still doesn't support CJK, since there is no usable CJK tokenizer at the moment. + return character.NewCharacterTokenizer(func(r rune) bool { + return unicode.IsLetter(r) || unicode.IsNumber(r) + }), nil +} + +const codeTokenFilterName = "codeTokenFilter" + +type codeTokenFilter struct { + re *regexp.Regexp +} + +func (c codeTokenFilter) Filter(stream analysis.TokenStream) (ret analysis.TokenStream) { + // split one token to "letter" parts and "number" parts (to keep the old behavior). + // e.g.: input token="port123", then the output tokens are "port123", "port", "123" + for _, token := range stream { + ret = append(ret, token) + m := c.re.FindAllIndex(token.Term, -1) + if len(m) > 1 { + for _, it := range m { + p1, p2 := it[0], it[1] + t := &analysis.Token{ + Start: token.Start + p1, + End: token.Start + p2, + Term: token.Term[p1:p2], + Position: token.Position, + Type: analysis.AlphaNumeric, + } + ret = append(ret, t) + } + } + } + return ret +} + +func codeTokenFilterConstructor(_ map[string]any, _ *registry.Cache) (analysis.TokenFilter, error) { + return &codeTokenFilter{ + re: regexp.MustCompile("[a-zA-Z]+|[0-9]+"), + }, nil +} + +func init() { + util.MustNoError(registry.RegisterTokenizer(codeTokenizerName, codeTokenizerConstructor)) + util.MustNoError(registry.RegisterTokenFilter(codeTokenFilterName, codeTokenFilterConstructor)) +} diff --git a/modules/indexer/code/bleve/token/path/path.go b/modules/indexer/code/bleve/token/path/path.go index e00bb695e26..09dbe0ad4dd 100644 --- a/modules/indexer/code/bleve/token/path/path.go +++ b/modules/indexer/code/bleve/token/path/path.go @@ -7,13 +7,13 @@ import ( "slices" "strings" + "gitea.dev/modules/util" + "github.com/blevesearch/bleve/v2/analysis" "github.com/blevesearch/bleve/v2/registry" ) -const ( - Name = "gitea/path" -) +const Name = "gitea/path" type TokenFilter struct{} @@ -98,8 +98,5 @@ func generatePathTokens(input analysis.TokenStream, reversed bool) analysis.Toke func init() { // FIXME: move it to the bleve's init function, but do not call it in global init - err := registry.RegisterTokenFilter(Name, TokenFilterConstructor) - if err != nil { - panic(err) - } + util.MustNoError(registry.RegisterTokenFilter(Name, TokenFilterConstructor)) } diff --git a/modules/indexer/issues/bleve/bleve.go b/modules/indexer/issues/bleve/bleve.go index 25dbdebfa47..02d1562b7fc 100644 --- a/modules/indexer/issues/bleve/bleve.go +++ b/modules/indexer/issues/bleve/bleve.go @@ -15,7 +15,6 @@ import ( "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" - "github.com/blevesearch/bleve/v2/analysis/token/camelcase" "github.com/blevesearch/bleve/v2/analysis/token/lowercase" "github.com/blevesearch/bleve/v2/analysis/token/unicodenorm" "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" @@ -26,7 +25,7 @@ import ( const ( issueIndexerAnalyzer = "issueIndexer" issueIndexerDocType = "issueIndexerDocType" - issueIndexerLatestVersion = 7 + issueIndexerLatestVersion = 8 ) const unicodeNormalizeName = "unicodeNormalize" @@ -103,7 +102,7 @@ func generateIssueIndexMapping() (mapping.IndexMapping, error) { "type": custom.Name, "char_filters": []string{}, "tokenizer": unicode.Name, - "token_filters": []string{unicodeNormalizeName, camelcase.Name, lowercase.Name}, + "token_filters": []string{unicodeNormalizeName, camelCaseKeepWholeName, lowercase.Name}, }); err != nil { return nil, err } diff --git a/modules/indexer/issues/bleve/bleve_test.go b/modules/indexer/issues/bleve/bleve_test.go index 7dc664e0efd..2900d1f9721 100644 --- a/modules/indexer/issues/bleve/bleve_test.go +++ b/modules/indexer/issues/bleve/bleve_test.go @@ -77,6 +77,47 @@ func TestBleveIndexerNoAssignee(t *testing.T) { } } +func TestBleveIndexerTokenFilter(t *testing.T) { + dir := t.TempDir() + indexer := NewIndexer(dir) + defer indexer.Close() + + _, err := indexer.Init(t.Context()) + require.NoError(t, err) + + require.NoError(t, indexer.Index(t.Context(), + &internal.IndexerData{ID: 1, Title: "fix(packages): SomeThing needs a rewrite (#12345)"}, + &internal.IndexerData{ID: 2, Title: "add support for mDNS discovery abc1234"}, + )) + + testCases := []struct { + name string + keyword string + expectedIDs []int64 + }{ + {name: "exact original case", keyword: "SomeThing", expectedIDs: []int64{1}}, + {name: "case matching original transitions", keyword: "someThing", expectedIDs: []int64{1}}, + {name: "all lower case", keyword: "something", expectedIDs: []int64{1}}, + {name: "all upper case", keyword: "SOMETHING", expectedIDs: []int64{1}}, + {name: "number match", keyword: "12345", expectedIDs: []int64{1}}, + {name: "number as part", keyword: "1234", expectedIDs: []int64{2}}, + {name: "sub-word search still works", keyword: "DNS", expectedIDs: []int64{2}}, + {name: "sub-word search, lower case", keyword: "mdns", expectedIDs: []int64{2}}, + {name: "keyword is camel case", keyword: "addSupport", expectedIDs: []int64{2}}, + {name: "keyword not match", keyword: "addsupport", expectedIDs: []int64{}}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + result, err := indexer.Search(t.Context(), &internal.SearchOptions{ + Keyword: testCase.keyword, + }) + require.NoError(t, err) + assert.ElementsMatch(t, testCase.expectedIDs, searchResultIDs(result)) + }) + } +} + func searchResultIDs(result *internal.SearchResult) []int64 { ids := make([]int64, 0, len(result.Hits)) for _, hit := range result.Hits { diff --git a/modules/indexer/issues/bleve/camelcase_keep_whole.go b/modules/indexer/issues/bleve/camelcase_keep_whole.go new file mode 100644 index 00000000000..af7cadb6222 --- /dev/null +++ b/modules/indexer/issues/bleve/camelcase_keep_whole.go @@ -0,0 +1,64 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package bleve + +import ( + "gitea.dev/modules/util" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/camelcase" + "github.com/blevesearch/bleve/v2/registry" +) + +const camelCaseKeepWholeName = "camelCaseKeepWhole" + +// camelCaseKeepWholeFilter behaves like bleve's built-in "camelCase" token filter, +// it also uses the whole word for a token. For example: when indexing "someThing", +// CamelCaseFilter only emits "some" and "thing", this filter also emits "something". +// It is questionable why the "issue indexer" used the CamelCaseFilter, it just can't search "someThing". +// To avoid breaking existing user experiences, this "whole token filter" is introduced to make the full word can be searched. +type camelCaseKeepWholeFilter struct { + inner *camelcase.CamelCaseFilter +} + +func (f *camelCaseKeepWholeFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + // First, do exactly what the stock camelCase filter does: split by "camelCase" tokens + split := f.inner.Filter(input) + + // Index the resulting position of the *first* sub-token produced for + // each original token (matched by start offset), so the duplicated + // whole-word token we add below lines up at the same position as the + // sub-word it stands in for, instead of drifting out of sync for + // fields with more than one original token. + posByStart := make(map[int]int, len(split)) + for _, tok := range split { + if _, ok := posByStart[tok.Start]; !ok { + posByStart[tok.Start] = tok.Position + } + } + + rv := make(analysis.TokenStream, 0, len(split)+len(input)) + rv = append(rv, split...) + + // Then append one extra, un-split copy of every original token, so the + // whole word survives as a standalone, independently searchable term. + for _, token := range input { + dup := *token + dup.Term = append([]byte(nil), token.Term...) + if pos, ok := posByStart[token.Start]; ok { + dup.Position = pos + } + rv = append(rv, &dup) + } + + return rv +} + +func camelCaseKeepWholeFilterConstructor(_ map[string]any, _ *registry.Cache) (analysis.TokenFilter, error) { + return &camelCaseKeepWholeFilter{inner: camelcase.NewCamelCaseFilter()}, nil +} + +func init() { + util.MustNoError(registry.RegisterTokenFilter(camelCaseKeepWholeName, camelCaseKeepWholeFilterConstructor)) +} diff --git a/modules/util/util.go b/modules/util/util.go index c8e1968bd45..13816182a3a 100644 --- a/modules/util/util.go +++ b/modules/util/util.go @@ -317,3 +317,9 @@ func DiffSlice[T comparable](oldSlice, newSlice []T) (added, removed []T) { } return added, removed } + +func MustNoError(err error) { + if err != nil { + panic(err) + } +} diff --git a/tests/integration/repo_search_test.go b/tests/integration/repo_search_test.go index 1b396ed2386..fa24fa09e81 100644 --- a/tests/integration/repo_search_test.go +++ b/tests/integration/repo_search_test.go @@ -53,9 +53,9 @@ func TestSearchRepo(t *testing.T) { } func testSearch(t *testing.T, url string, expected []string) { + t.Helper() req := NewRequest(t, "GET", url) resp := MakeRequest(t, req, http.StatusOK) - filenames := resultFilenames(NewHTMLParser(t, resp.Body)) - assert.Equal(t, expected, filenames) + assert.Equal(t, expected, filenames, "url=%s", url) }