mirror of
https://github.com/go-gitea/gitea.git
synced 2026-10-29 20:31:54 +00:00
1ebeade2c1
Code search now matches case-insensitive literal substrings like GitHub, in every script. Word tokenizers can't do this: the ES pattern dropped every non-ASCII word, and the `standard` tokenizer keeps `console.log` as one token. - Elasticsearch indexes positional trigrams, plus 1-2 grams for short terms - Bleve indexes distinct 1-3 grams as a candidate filter and verifies candidates against the content before paging, because phrase queries over trigrams are quadratic in repeated grams - Snippet offsets come from the original content, so `<em>` in sources and width-changing lowercase no longer misplace them - `search_mode=fuzzy` requests behave like Words, the UI never offered fuzzy for code search - The ES code indexer test now runs in CI, it was skipped before There are no UI changes. Both code indexes rebuild after upgrade. Fixes: https://github.com/go-gitea/gitea/issues/40027 Fixes: https://github.com/go-gitea/gitea/issues/16158 Fixes: https://github.com/go-gitea/gitea/issues/21359 --------- Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
42 lines
1.2 KiB
Go
42 lines
1.2 KiB
Go
// Copyright 2026 The Gitea Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package bleve
|
|
|
|
import (
|
|
"bytes"
|
|
"unicode/utf8"
|
|
|
|
"github.com/blevesearch/bleve/v2/analysis"
|
|
"github.com/blevesearch/bleve/v2/registry"
|
|
)
|
|
|
|
const codeTokenizerName = "codeTokenizer"
|
|
|
|
// codeTokenizer emits every distinct lowercase gram of 1 to 3 runes once, search verifies longer terms against the content
|
|
type codeTokenizer struct{}
|
|
|
|
func (codeTokenizer) Tokenize(input []byte) analysis.TokenStream {
|
|
lower := bytes.ToLower(input)
|
|
seen := make(map[string]bool)
|
|
var tokens analysis.TokenStream
|
|
var starts [3]int
|
|
for end, count := 0, 0; end < len(lower); {
|
|
starts[0], starts[1], starts[2] = starts[1], starts[2], end
|
|
count = min(count+1, len(starts))
|
|
_, size := utf8.DecodeRune(lower[end:])
|
|
end += size
|
|
for _, start := range starts[len(starts)-count:] {
|
|
if gram := lower[start:end]; !seen[string(gram)] {
|
|
seen[string(gram)] = true
|
|
tokens = append(tokens, &analysis.Token{Start: start, End: end, Term: gram, Position: len(tokens) + 1, Type: analysis.AlphaNumeric})
|
|
}
|
|
}
|
|
}
|
|
return tokens
|
|
}
|
|
|
|
func codeTokenizerConstructor(_config map[string]any, _cache *registry.Cache) (analysis.Tokenizer, error) {
|
|
return codeTokenizer{}, nil
|
|
}
|