Gitea-John71/Gitea
2
0
Fork 0
mirror of https://github.com/go-gitea/gitea.git synced 2026-08-17 21:51:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(indexer): correct bleve indexer token filters (#38853)

Browse Source 
* fix #36228
* fix #37221

---------

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
This commit is contained in:
Mitrahsoft
2026-08-17 13:29:41 +05:30
committed by GitHub
parent 3cd6672e48
commit 1b21c8a1e6
9 changed files with 246 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/indexer/code/bleve/bleve.go
+3 -5
View File
@@ -32,7 +32,6 @@ import (
analyzer_keyword "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
"github.com/blevesearch/bleve/v2/analysis/token/unicodenorm"
"github.com/blevesearch/bleve/v2/analysis/tokenizer/letter"
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
"github.com/blevesearch/bleve/v2/mapping"
"github.com/blevesearch/bleve/v2/search/query"
@@ -69,9 +68,8 @@ func (d *RepoIndexerData) Type() string {
const (
repoIndexerAnalyzer = "repoIndexerAnalyzer"
filenameIndexerAnalyzer = "filenameIndexerAnalyzer"
filenameIndexerTokenizer = "filenameIndexerTokenizer"
repoIndexerDocType = "repoIndexerDocType"
repoIndexerLatestVersion = 9
repoIndexerLatestVersion = 10
)
// generateBleveIndexMapping generates a bleve index mapping for the repo indexer
@@ -107,8 +105,8 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) {
} else if err := mapping.AddCustomAnalyzer(repoIndexerAnalyzer, map[string]any{
"type": analyzer_custom.Name,
"char_filters": []string{},
"tokenizer": letter.Name,
"token_filters": []string{unicodeNormalizeName, lowercase.Name},
"tokenizer": codeTokenizerName,
"token_filters": []string{unicodeNormalizeName, codeTokenFilterName, lowercase.Name},
}); err != nil {
return nil, err
}
modules/indexer/code/bleve/bleve_test.go
+59
View File
@@ -0,0 +1,59 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package bleve
import (
"testing"
"time"
"gitea.dev/models/db"
"gitea.dev/modules/indexer/code/internal"
inner_bleve "gitea.dev/modules/indexer/internal/bleve"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestBleveIndexerTokenFilter(t *testing.T) {
dir := t.TempDir()
indexer := NewIndexer(dir)
defer indexer.Close()
_, err := indexer.Init(t.Context())
require.NoError(t, err)
batch := inner_bleve.NewFlushingBatch(indexer.inner.Indexer, maxBatchSize)
batch.Index("2", &RepoIndexerData{RepoID: 2, Content: "mDNS.port2=12345", UpdatedAt: time.Now()})
batch.Flush()
testCases := []struct {
keyword string
expectedIDs []int64
}{
{keyword: "12345", expectedIDs: []int64{2}},
{keyword: "DNS", expectedIDs: []int64{}},
{keyword: "mdns", expectedIDs: []int64{2}},
{keyword: "port", expectedIDs: []int64{2}},
{keyword: "port2", expectedIDs: []int64{2}},
}
for _, testCase := range testCases {
t.Run(testCase.keyword, func(t *testing.T) {
_, results, _, err := indexer.Search(t.Context(), &internal.SearchOptions{
Paginator: &db.ListOptions{Page: 1, PageSize: 1},
Keyword: testCase.keyword,
})
require.NoError(t, err)
assert.ElementsMatch(t, testCase.expectedIDs, searchResultIDs(results))
})
}
}
func searchResultIDs(result []*internal.SearchResult) []int64 {
ids := make([]int64, 0, len(result))
for _, hit := range result {
ids = append(ids, hit.RepoID)
}
return ids
}
modules/indexer/code/bleve/code_token.go
+65
View File
@@ -0,0 +1,65 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package bleve
import (
"regexp"
"unicode"
"gitea.dev/modules/util"
"github.com/blevesearch/bleve/v2/analysis"
"github.com/blevesearch/bleve/v2/analysis/tokenizer/character"
"github.com/blevesearch/bleve/v2/registry"
)
const codeTokenizerName = "codeTokenizer"
func codeTokenizerConstructor(_ map[string]any, _ *registry.Cache) (analysis.Tokenizer, error) {
// Old code used "letter" tokenizer which doesn't support CJK.
// Here it still doesn't support CJK, since there is no usable CJK tokenizer at the moment.
return character.NewCharacterTokenizer(func(r rune) bool {
return unicode.IsLetter(r) || unicode.IsNumber(r)
}), nil
}
const codeTokenFilterName = "codeTokenFilter"
type codeTokenFilter struct {
re *regexp.Regexp
}
func (c codeTokenFilter) Filter(stream analysis.TokenStream) (ret analysis.TokenStream) {
// split one token to "letter" parts and "number" parts (to keep the old behavior).
// e.g.: input token="port123", then the output tokens are "port123", "port", "123"
for _, token := range stream {
ret = append(ret, token)
m := c.re.FindAllIndex(token.Term, -1)
if len(m) > 1 {
for _, it := range m {
p1, p2 := it[0], it[1]
t := &analysis.Token{
Start: token.Start + p1,
End: token.Start + p2,
Term: token.Term[p1:p2],
Position: token.Position,
Type: analysis.AlphaNumeric,
}
ret = append(ret, t)
}
}
}
return ret
}
func codeTokenFilterConstructor(_ map[string]any, _ *registry.Cache) (analysis.TokenFilter, error) {
return &codeTokenFilter{
re: regexp.MustCompile("[a-zA-Z]+|[0-9]+"),
}, nil
}
func init() {
util.MustNoError(registry.RegisterTokenizer(codeTokenizerName, codeTokenizerConstructor))
util.MustNoError(registry.RegisterTokenFilter(codeTokenFilterName, codeTokenFilterConstructor))
}
modules/indexer/code/bleve/token/path/path.go
+4 -7
View File
@@ -7,13 +7,13 @@ import (
"slices"
"strings"
"gitea.dev/modules/util"
"github.com/blevesearch/bleve/v2/analysis"
"github.com/blevesearch/bleve/v2/registry"
)
const (
Name = "gitea/path"
)
const Name = "gitea/path"
type TokenFilter struct{}
@@ -98,8 +98,5 @@ func generatePathTokens(input analysis.TokenStream, reversed bool) analysis.Toke
func init() {
// FIXME: move it to the bleve's init function, but do not call it in global init
err := registry.RegisterTokenFilter(Name, TokenFilterConstructor)
if err != nil {
panic(err)
}
util.MustNoError(registry.RegisterTokenFilter(Name, TokenFilterConstructor))
}
modules/indexer/issues/bleve/bleve.go
+2 -3
View File
@@ -15,7 +15,6 @@ import (
"github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
"github.com/blevesearch/bleve/v2/analysis/token/camelcase"
"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
"github.com/blevesearch/bleve/v2/analysis/token/unicodenorm"
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
@@ -26,7 +25,7 @@ import (
const (
issueIndexerAnalyzer = "issueIndexer"
issueIndexerDocType = "issueIndexerDocType"
issueIndexerLatestVersion = 7
issueIndexerLatestVersion = 8
)
const unicodeNormalizeName = "unicodeNormalize"
@@ -103,7 +102,7 @@ func generateIssueIndexMapping() (mapping.IndexMapping, error) {
"type": custom.Name,
"char_filters": []string{},
"tokenizer": unicode.Name,
"token_filters": []string{unicodeNormalizeName, camelcase.Name, lowercase.Name},
"token_filters": []string{unicodeNormalizeName, camelCaseKeepWholeName, lowercase.Name},
}); err != nil {
return nil, err
}
modules/indexer/issues/bleve/bleve_test.go
+41
View File
@@ -77,6 +77,47 @@ func TestBleveIndexerNoAssignee(t *testing.T) {
}
}
func TestBleveIndexerTokenFilter(t *testing.T) {
dir := t.TempDir()
indexer := NewIndexer(dir)
defer indexer.Close()
_, err := indexer.Init(t.Context())
require.NoError(t, err)
require.NoError(t, indexer.Index(t.Context(),
&internal.IndexerData{ID: 1, Title: "fix(packages): SomeThing needs a rewrite (#12345)"},
&internal.IndexerData{ID: 2, Title: "add support for mDNS discovery abc1234"},
))
testCases := []struct {
name string
keyword string
expectedIDs []int64
}{
{name: "exact original case", keyword: "SomeThing", expectedIDs: []int64{1}},
{name: "case matching original transitions", keyword: "someThing", expectedIDs: []int64{1}},
{name: "all lower case", keyword: "something", expectedIDs: []int64{1}},
{name: "all upper case", keyword: "SOMETHING", expectedIDs: []int64{1}},
{name: "number match", keyword: "12345", expectedIDs: []int64{1}},
{name: "number as part", keyword: "1234", expectedIDs: []int64{2}},
{name: "sub-word search still works", keyword: "DNS", expectedIDs: []int64{2}},
{name: "sub-word search, lower case", keyword: "mdns", expectedIDs: []int64{2}},
{name: "keyword is camel case", keyword: "addSupport", expectedIDs: []int64{2}},
{name: "keyword not match", keyword: "addsupport", expectedIDs: []int64{}},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
result, err := indexer.Search(t.Context(), &internal.SearchOptions{
Keyword: testCase.keyword,
})
require.NoError(t, err)
assert.ElementsMatch(t, testCase.expectedIDs, searchResultIDs(result))
})
}
}
func searchResultIDs(result *internal.SearchResult) []int64 {
ids := make([]int64, 0, len(result.Hits))
for _, hit := range result.Hits {
modules/indexer/issues/bleve/camelcase_keep_whole.go
+64
View File
@@ -0,0 +1,64 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package bleve
import (
"gitea.dev/modules/util"
"github.com/blevesearch/bleve/v2/analysis"
"github.com/blevesearch/bleve/v2/analysis/token/camelcase"
"github.com/blevesearch/bleve/v2/registry"
)
const camelCaseKeepWholeName = "camelCaseKeepWhole"
// camelCaseKeepWholeFilter behaves like bleve's built-in "camelCase" token filter,
// it also uses the whole word for a token. For example: when indexing "someThing",
// CamelCaseFilter only emits "some" and "thing", this filter also emits "something".
// It is questionable why the "issue indexer" used the CamelCaseFilter, it just can't search "someThing".
// To avoid breaking existing user experiences, this "whole token filter" is introduced to make the full word can be searched.
type camelCaseKeepWholeFilter struct {
inner *camelcase.CamelCaseFilter
}
func (f *camelCaseKeepWholeFilter) Filter(input analysis.TokenStream) analysis.TokenStream {
// First, do exactly what the stock camelCase filter does: split by "camelCase" tokens
split := f.inner.Filter(input)
// Index the resulting position of the *first* sub-token produced for
// each original token (matched by start offset), so the duplicated
// whole-word token we add below lines up at the same position as the
// sub-word it stands in for, instead of drifting out of sync for
// fields with more than one original token.
posByStart := make(map[int]int, len(split))
for _, tok := range split {
if _, ok := posByStart[tok.Start]; !ok {
posByStart[tok.Start] = tok.Position
}
}
rv := make(analysis.TokenStream, 0, len(split)+len(input))
rv = append(rv, split...)
// Then append one extra, un-split copy of every original token, so the
// whole word survives as a standalone, independently searchable term.
for _, token := range input {
dup := *token
dup.Term = append([]byte(nil), token.Term...)
if pos, ok := posByStart[token.Start]; ok {
dup.Position = pos
}
rv = append(rv, &dup)
}
return rv
}
func camelCaseKeepWholeFilterConstructor(_ map[string]any, _ *registry.Cache) (analysis.TokenFilter, error) {
return &camelCaseKeepWholeFilter{inner: camelcase.NewCamelCaseFilter()}, nil
}
func init() {
util.MustNoError(registry.RegisterTokenFilter(camelCaseKeepWholeName, camelCaseKeepWholeFilterConstructor))
}
modules/util/util.go
+6
View File
@@ -317,3 +317,9 @@ func DiffSlice[T comparable](oldSlice, newSlice []T) (added, removed []T) {
}
return added, removed
}
func MustNoError(err error) {
if err != nil {
panic(err)
}
}
tests/integration/repo_search_test.go
+2 -2
View File
@@ -53,9 +53,9 @@ func TestSearchRepo(t *testing.T) {
}
func testSearch(t *testing.T, url string, expected []string) {
t.Helper()
req := NewRequest(t, "GET", url)
resp := MakeRequest(t, req, http.StatusOK)
filenames := resultFilenames(NewHTMLParser(t, resp.Body))
assert.Equal(t, expected, filenames)
assert.Equal(t, expected, filenames, "url=%s", url)
}