From 1ebeade2c111bb48f8af93befc953b73261acd29 Mon Sep 17 00:00:00 2001 From: Danny Ward <71604164+dannyward630@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:38:45 -0400 Subject: [PATCH] fix(indexer): search code as case-insensitive literal substrings (#40028) Code search now matches case-insensitive literal substrings like GitHub, in every script. Word tokenizers can't do this: the ES pattern dropped every non-ASCII word, and the `standard` tokenizer keeps `console.log` as one token. - Elasticsearch indexes positional trigrams, plus 1-2 grams for short terms - Bleve indexes distinct 1-3 grams as a candidate filter and verifies candidates against the content before paging, because phrase queries over trigrams are quadratic in repeated grams - Snippet offsets come from the original content, so `` in sources and width-changing lowercase no longer misplace them - `search_mode=fuzzy` requests behave like Words, the UI never offered fuzzy for code search - The ES code indexer test now runs in CI, it was skipped before There are no UI changes. Both code indexes rebuild after upgrade. Fixes: https://github.com/go-gitea/gitea/issues/40027 Fixes: https://github.com/go-gitea/gitea/issues/16158 Fixes: https://github.com/go-gitea/gitea/issues/21359 --------- Co-authored-by: silverwind Co-authored-by: wxiaoguang --- .github/workflows/pull-db-tests.yml | 1 - modules/indexer/code/bleve/bleve.go | 163 +++++++++--------- modules/indexer/code/bleve/bleve_test.go | 37 +++- modules/indexer/code/bleve/token_code.go | 59 +++---- .../code/elasticsearch/elasticsearch.go | 97 +++++------ .../code/elasticsearch/elasticsearch_test.go | 16 -- modules/indexer/code/indexer_test.go | 65 +++++-- modules/indexer/code/internal/util.go | 43 +++++ .../org42/search-by-path.git/description | 9 - .../search-by-path.git/hooks/post-receive | 7 - .../hooks/post-receive.d/gitea | 2 - .../search-by-path.git/hooks/pre-receive | 7 - .../hooks/pre-receive.d/gitea | 2 - .../search-by-path.git/hooks/proc-receive | 7 - .../hooks/proc-receive.d/gitea | 2 - .../org42/search-by-path.git/hooks/update | 7 - .../search-by-path.git/hooks/update.d/gitea | 2 - .../org42/search-by-path.git/info/refs | 13 -- .../org42/search-by-path.git/objects/.keep | 0 .../search-by-path.git/objects/info/packs | 2 - ...76cf6e2b46bc816936ab69306fb10aea571.bitmap | Bin 678 -> 0 bytes ...bef76cf6e2b46bc816936ab69306fb10aea571.idx | Bin 2108 -> 0 bytes ...ef76cf6e2b46bc816936ab69306fb10aea571.pack | Bin 6545 -> 0 bytes ...bef76cf6e2b46bc816936ab69306fb10aea571.rev | Bin 200 -> 0 bytes .../org42/search-by-path.git/packed-refs | 14 -- .../user2/glob.git/objects/.keep | 0 .../48/06cb9df135782b818c968c2fadbd2c150d23d6 | Bin 21 -> 0 bytes .../59/fee614e09d1f1cd1e15e4b2a7e9c8873a81498 | Bin 34 -> 0 bytes .../7c/8ac2f8d82a1eb5f6aaece6629ff11015f91eb4 | Bin 21 -> 0 bytes .../8e/592e636d27ac144f92f7fe8c33631cbdea594d | Bin 78 -> 0 bytes .../95/aff026f99a9ab76fbd01decb63dd3dbc03e498 | Bin 34 -> 0 bytes .../ae/d1ffed24cc3cf9b80490795e893cae4bddd684 | Bin 108 -> 0 bytes .../bf/d6a6583f9a9ac59bd726c1df26c64a89427ede | Bin 21 -> 0 bytes .../c8/eb3b6c767ccb68411d0a1f6c769be69fb4d95a | 1 - .../de/6be43fe8eb19ca3f4e934cb8b9a9a0b20fe865 | Bin 50 -> 0 bytes .../ef/6b814b610d8e7717aa0f71fbe5842bcf814697 | 2 - .../user2/glob.git/refs/.keep | 0 .../user2/glob.git/refs/heads/master | 1 - tests/integration/links_test.go | 29 ++-- tests/integration/repo_archive_test.go | 36 ++-- tests/integration/repo_search_test.go | 23 ++- 41 files changed, 330 insertions(+), 317 deletions(-) delete mode 100644 modules/indexer/code/elasticsearch/elasticsearch_test.go delete mode 100644 tests/gitea-repositories-meta/org42/search-by-path.git/description delete mode 100755 tests/gitea-repositories-meta/org42/search-by-path.git/hooks/post-receive delete mode 100755 tests/gitea-repositories-meta/org42/search-by-path.git/hooks/post-receive.d/gitea delete mode 100755 tests/gitea-repositories-meta/org42/search-by-path.git/hooks/pre-receive delete mode 100755 tests/gitea-repositories-meta/org42/search-by-path.git/hooks/pre-receive.d/gitea delete mode 100755 tests/gitea-repositories-meta/org42/search-by-path.git/hooks/proc-receive delete mode 100755 tests/gitea-repositories-meta/org42/search-by-path.git/hooks/proc-receive.d/gitea delete mode 100755 tests/gitea-repositories-meta/org42/search-by-path.git/hooks/update delete mode 100755 tests/gitea-repositories-meta/org42/search-by-path.git/hooks/update.d/gitea delete mode 100644 tests/gitea-repositories-meta/org42/search-by-path.git/info/refs create mode 100644 tests/gitea-repositories-meta/org42/search-by-path.git/objects/.keep delete mode 100644 tests/gitea-repositories-meta/org42/search-by-path.git/objects/info/packs delete mode 100644 tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.bitmap delete mode 100644 tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.idx delete mode 100644 tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.pack delete mode 100644 tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.rev delete mode 100644 tests/gitea-repositories-meta/org42/search-by-path.git/packed-refs create mode 100644 tests/gitea-repositories-meta/user2/glob.git/objects/.keep delete mode 100644 tests/gitea-repositories-meta/user2/glob.git/objects/48/06cb9df135782b818c968c2fadbd2c150d23d6 delete mode 100644 tests/gitea-repositories-meta/user2/glob.git/objects/59/fee614e09d1f1cd1e15e4b2a7e9c8873a81498 delete mode 100644 tests/gitea-repositories-meta/user2/glob.git/objects/7c/8ac2f8d82a1eb5f6aaece6629ff11015f91eb4 delete mode 100644 tests/gitea-repositories-meta/user2/glob.git/objects/8e/592e636d27ac144f92f7fe8c33631cbdea594d delete mode 100644 tests/gitea-repositories-meta/user2/glob.git/objects/95/aff026f99a9ab76fbd01decb63dd3dbc03e498 delete mode 100644 tests/gitea-repositories-meta/user2/glob.git/objects/ae/d1ffed24cc3cf9b80490795e893cae4bddd684 delete mode 100644 tests/gitea-repositories-meta/user2/glob.git/objects/bf/d6a6583f9a9ac59bd726c1df26c64a89427ede delete mode 100644 tests/gitea-repositories-meta/user2/glob.git/objects/c8/eb3b6c767ccb68411d0a1f6c769be69fb4d95a delete mode 100644 tests/gitea-repositories-meta/user2/glob.git/objects/de/6be43fe8eb19ca3f4e934cb8b9a9a0b20fe865 delete mode 100644 tests/gitea-repositories-meta/user2/glob.git/objects/ef/6b814b610d8e7717aa0f71fbe5842bcf814697 create mode 100644 tests/gitea-repositories-meta/user2/glob.git/refs/.keep delete mode 100644 tests/gitea-repositories-meta/user2/glob.git/refs/heads/master diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index 56dc4733abe..db5845864e6 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -215,7 +215,6 @@ jobs: run: GITEA_TEST_DATABASE=mysql make test-integration env: TAGS: bindata - TEST_INDEXER_CODE_ES_URL: "http://elastic:changeme@elasticsearch:9200" test-mssql: if: needs.files-changed.outputs.backend == 'true' diff --git a/modules/indexer/code/bleve/bleve.go b/modules/indexer/code/bleve/bleve.go index d22f735571a..bcacef245b4 100644 --- a/modules/indexer/code/bleve/bleve.go +++ b/modules/indexer/code/bleve/bleve.go @@ -7,9 +7,11 @@ import ( "context" "fmt" "io" + "slices" "strconv" "strings" "time" + "unicode/utf8" repo_model "gitea.dev/models/repo" "gitea.dev/modules/analyze" @@ -68,7 +70,7 @@ const ( repoIndexerAnalyzer = "repoIndexerAnalyzer" filenameIndexerAnalyzer = "filenameIndexerAnalyzer" repoIndexerDocType = "repoIndexerDocType" - repoIndexerLatestVersion = 11 + repoIndexerLatestVersion = 12 ) // generateBleveIndexMapping generates a bleve index mapping for the repo indexer @@ -80,6 +82,8 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) { textFieldMapping := bleve.NewTextFieldMapping() textFieldMapping.IncludeInAll = false + textFieldMapping.DocValues = false + textFieldMapping.IncludeTermVectors = false docMapping.AddFieldMappingsAt("Content", textFieldMapping) fileNamedMapping := bleve.NewTextFieldMapping() @@ -105,7 +109,7 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) { "type": analyzer_custom.Name, "char_filters": []string{}, "tokenizer": codeTokenizerName, - "token_filters": []string{unicodeNormalizeName, codeTokenFilterName, lowercase.Name}, + "token_filters": []string{}, }); err != nil { return nil, err } @@ -141,7 +145,6 @@ func (b *Indexer) SupportedSearchModes() []indexer.SearchMode { func init() { // due to bleve's design problem, the "Register" must be done in the main goroutine, otherwise data-race util.MustNoError(registry.RegisterTokenizer(codeTokenizerName, codeTokenizerConstructor)) - util.MustNoError(registry.RegisterTokenFilter(codeTokenFilterName, codeTokenFilterConstructor)) util.MustNoError(registry.RegisterTokenizer(pathTokenizerName, pathTokenizerConstructor)) } @@ -266,25 +269,15 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int pathQuery.FieldVal = "Filename" pathQuery.SetBoost(10) - searchMode := util.IfZero(opts.SearchMode, b.SupportedSearchModes()[0].ModeValue) - if searchMode == indexer.SearchModeExact { - // 1.21 used NewPrefixQuery, but it seems not working well, and later releases changed to NewMatchPhraseQuery - q := bleve.NewMatchPhraseQuery(opts.Keyword) - q.Analyzer = repoIndexerAnalyzer - q.FieldVal = "Content" - contentQuery = q - } else /* words */ { - q := bleve.NewMatchQuery(opts.Keyword) - q.FieldVal = "Content" - q.Analyzer = repoIndexerAnalyzer - if searchMode == indexer.SearchModeFuzzy { - // this logic doesn't seem right, it is only used to pass the test-case `Keyword: "dESCRIPTION"`, which doesn't seem to be a real-life use-case. - q.Fuzziness = inner_bleve.GuessFuzzinessByKeyword(opts.Keyword) - } else { - q.Operator = query.MatchQueryOperatorAnd - } - contentQuery = q + terms := internal.SearchTerms(opts) + if len(terms) == 0 { + return 0, nil, nil, nil } + contentQueries := make([]query.Query, 0, len(terms)) + for _, term := range terms { + contentQueries = append(contentQueries, inner_bleve.MatchAndQuery(term, "Content", repoIndexerAnalyzer, 0)) + } + contentQuery = bleve.NewConjunctionQuery(contentQueries...) keywordQuery = bleve.NewDisjunctionQuery(contentQuery, pathQuery) @@ -316,67 +309,79 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int } from, pageSize := opts.GetSkipTake() - searchRequest := bleve.NewSearchRequestOptions(indexerQuery, pageSize, from, false) - searchRequest.Fields = []string{"Content", "Filename", "RepoID", "Language", "CommitID", "UpdatedAt"} - searchRequest.IncludeLocations = true - - if len(opts.Language) == 0 { - searchRequest.AddFacet("languages", bleve.NewFacetRequest("Language", 10)) + // grams prove terms of up to 3 runes, longer ones are verified against the content, so fetch candidates until the page is full + matched, scanned := 0, 0 + if !slices.ContainsFunc(terms, func(term string) bool { return utf8.RuneCountInString(term) > 3 }) { + matched, scanned = from, from } + var result *bleve.SearchResult + var total int64 + searchResults := make([]*internal.SearchResult, 0, pageSize) + for { + size := from + pageSize - matched + searchRequest := bleve.NewSearchRequestOptions(indexerQuery, size, scanned, false) + searchRequest.Fields = []string{"Content", "Filename", "RepoID", "Language", "CommitID", "UpdatedAt"} + searchRequest.IncludeLocations = true + if result == nil && len(opts.Language) == 0 { + searchRequest.AddFacet("languages", bleve.NewFacetRequest("Language", 10)) + } + searchRequest.SortBy([]string{"-_score", "UpdatedAt"}) - searchRequest.SortBy([]string{"-_score", "UpdatedAt"}) + batch, err := b.inner.Indexer.SearchInContext(ctx, searchRequest) + if err != nil { + return 0, nil, nil, err + } + if result == nil { + result = batch + } - result, err := b.inner.Indexer.SearchInContext(ctx, searchRequest) - if err != nil { - return 0, nil, nil, err + for _, hit := range batch.Hits { + scanned++ + content, okContent := hit.Fields["Content"].(string) + language, okLanguage := hit.Fields["Language"].(string) + commitID, okCommitID := hit.Fields["CommitID"].(string) + updatedAt, okUpdatedAt := hit.Fields["UpdatedAt"].(string) + repoID, okRepoID := hit.Fields["RepoID"].(float64) + if !okContent || !okLanguage || !okCommitID || !okUpdatedAt || !okRepoID { + hitFieldsJson, _ := json.Marshal(hit.Fields) + setting.PanicInDevOrTesting("unexpected field types in search hit %q: %s", hit.ID, string(hitFieldsJson)) + return 0, nil, nil, fmt.Errorf("unexpected field types in search hit %q", hit.ID) + } + + var startIndex, endIndex int + if len(hit.Locations["Filename"]) > 0 { + startIndex, endIndex = internal.FilenameMatchIndexPos(content) + } else if startIndex, endIndex = internal.ContentMatchIndexPos(content, terms); startIndex < 0 { + continue + } + matched++ + if matched <= from { + continue + } + + var updatedUnix timeutil.TimeStamp + if t, err := time.Parse(time.RFC3339, updatedAt); err == nil { + updatedUnix = timeutil.TimeStamp(t.Unix()) + } + searchResults = append(searchResults, &internal.SearchResult{ + RepoID: int64(repoID), + StartIndex: startIndex, + EndIndex: endIndex, + Filename: internal.FilenameOfIndexerID(hit.ID), + Content: content, + CommitID: commitID, + UpdatedUnix: updatedUnix, + Language: language, + Color: enry.GetColor(language), + }) + } + if matched == from+pageSize || len(batch.Hits) < size { + total = max(int64(matched), int64(matched)+int64(batch.Total)-int64(scanned)) + break + } } - - total := int64(result.Total) - - searchResults := make([]*internal.SearchResult, len(result.Hits)) - for i, hit := range result.Hits { - content, okContent := hit.Fields["Content"].(string) - language, okLanguage := hit.Fields["Language"].(string) - commitID, okCommitID := hit.Fields["CommitID"].(string) - updatedAt, okUpdatedAt := hit.Fields["UpdatedAt"].(string) - repoID, okRepoID := hit.Fields["RepoID"].(float64) - if !okContent || !okLanguage || !okCommitID || !okUpdatedAt || !okRepoID { - hitFieldsJson, _ := json.Marshal(hit.Fields) - setting.PanicInDevOrTesting("unexpected field types in search hit %q: %s", hit.ID, string(hitFieldsJson)) - return 0, nil, nil, fmt.Errorf("unexpected field types in search hit %q", hit.ID) - } - - startIndex, endIndex := -1, -1 - for _, locations := range hit.Locations["Content"] { - location := locations[0] - locationStart := int(location.Start) - locationEnd := int(location.End) - if startIndex < 0 || locationStart < startIndex { - startIndex = locationStart - } - if endIndex < 0 || locationEnd > endIndex { - endIndex = locationEnd - } - } - if len(hit.Locations["Filename"]) > 0 { - startIndex, endIndex = internal.FilenameMatchIndexPos(content) - } - - var updatedUnix timeutil.TimeStamp - if t, err := time.Parse(time.RFC3339, updatedAt); err == nil { - updatedUnix = timeutil.TimeStamp(t.Unix()) - } - searchResults[i] = &internal.SearchResult{ - RepoID: int64(repoID), - StartIndex: startIndex, - EndIndex: endIndex, - Filename: internal.FilenameOfIndexerID(hit.ID), - Content: content, - CommitID: commitID, - UpdatedUnix: updatedUnix, - Language: language, - Color: enry.GetColor(language), - } + if total == 0 { + return 0, nil, nil, nil } searchResultLanguages := make([]*internal.SearchResultLanguages, 0, 10) @@ -384,9 +389,9 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int // Use separate query to go get all language counts facetRequest := bleve.NewSearchRequestOptions(facetQuery, 1, 0, false) facetRequest.Fields = []string{"Content", "RepoID", "Language", "CommitID", "UpdatedAt"} - facetRequest.IncludeLocations = true facetRequest.AddFacet("languages", bleve.NewFacetRequest("Language", 10)) + var err error if result, err = b.inner.Indexer.Search(facetRequest); err != nil { return 0, nil, nil, err } diff --git a/modules/indexer/code/bleve/bleve_test.go b/modules/indexer/code/bleve/bleve_test.go index 0a59b677e13..06d55de64f4 100644 --- a/modules/indexer/code/bleve/bleve_test.go +++ b/modules/indexer/code/bleve/bleve_test.go @@ -4,12 +4,15 @@ package bleve import ( + "strconv" "testing" "time" "gitea.dev/models/db" + indexer_module "gitea.dev/modules/indexer" "gitea.dev/modules/indexer/code/internal" inner_bleve "gitea.dev/modules/indexer/internal/bleve" + "gitea.dev/modules/util" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -32,7 +35,7 @@ func TestBleveIndexerTokenFilter(t *testing.T) { expectedIDs []int64 }{ {keyword: "12345", expectedIDs: []int64{2}}, - {keyword: "DNS", expectedIDs: []int64{}}, + {keyword: "DNS", expectedIDs: []int64{2}}, {keyword: "mdns", expectedIDs: []int64{2}}, {keyword: "port", expectedIDs: []int64{2}}, {keyword: "port2", expectedIDs: []int64{2}}, @@ -54,6 +57,38 @@ func TestBleveIndexerTokenFilter(t *testing.T) { } } +func TestBleveIndexerPagesVerifiedCandidates(t *testing.T) { + indexer := NewIndexer(t.TempDir()) + defer indexer.Close() + _, err := indexer.Init(t.Context()) + require.NoError(t, err) + + batch := inner_bleve.NewFlushingBatch(indexer.inner.Indexer, maxBatchSize) + for i, content := range []string{"abc bcd cde def", "abcdef", "abc bcd cde def", "abcdef", "abc bcd cde def"} { + filename := strconv.Itoa(i) + require.NoError(t, batch.Index(internal.FilenameIndexerID(1, filename), &RepoIndexerData{RepoID: 1, Filename: filename, Content: content, UpdatedAt: time.Now()})) + } + require.NoError(t, batch.Flush()) + + var filenames []string + for page := 1; page <= 3; page++ { + total, results, _, err := indexer.Search(t.Context(), &internal.SearchOptions{ + Paginator: &db.ListOptions{Page: page, PageSize: 1}, + Keyword: "abcdef", + SearchMode: indexer_module.SearchModeExact, + }) + require.NoError(t, err) + require.Len(t, results, util.Iif(page < 3, 1, 0)) + if page == 3 { + assert.EqualValues(t, 2, total) + } + for _, result := range results { + filenames = append(filenames, result.Filename) + } + } + assert.ElementsMatch(t, []string{"1", "3"}, filenames) +} + func searchResultIDs(result []*internal.SearchResult) []int64 { ids := make([]int64, 0, len(result)) for _, hit := range result { diff --git a/modules/indexer/code/bleve/token_code.go b/modules/indexer/code/bleve/token_code.go index 53a6c401a0e..6a7d6d6d08b 100644 --- a/modules/indexer/code/bleve/token_code.go +++ b/modules/indexer/code/bleve/token_code.go @@ -4,55 +4,38 @@ package bleve import ( - "regexp" - "unicode" + "bytes" + "unicode/utf8" "github.com/blevesearch/bleve/v2/analysis" - "github.com/blevesearch/bleve/v2/analysis/tokenizer/character" "github.com/blevesearch/bleve/v2/registry" ) const codeTokenizerName = "codeTokenizer" -func codeTokenizerConstructor(_ map[string]any, _ *registry.Cache) (analysis.Tokenizer, error) { - // Old code used "letter" tokenizer which doesn't support CJK. - // Here it still doesn't support CJK, since there is no usable CJK tokenizer at the moment. - return character.NewCharacterTokenizer(func(r rune) bool { - return unicode.IsLetter(r) || unicode.IsNumber(r) - }), nil -} +// codeTokenizer emits every distinct lowercase gram of 1 to 3 runes once, search verifies longer terms against the content +type codeTokenizer struct{} -const codeTokenFilterName = "codeTokenFilter" - -type codeTokenFilter struct { - re *regexp.Regexp -} - -func (c codeTokenFilter) Filter(stream analysis.TokenStream) (ret analysis.TokenStream) { - // split one token to "letter" parts and "number" parts (to keep the old behavior). - // e.g.: input token="port123", then the output tokens are "port123", "port", "123" - for _, token := range stream { - ret = append(ret, token) - m := c.re.FindAllIndex(token.Term, -1) - if len(m) > 1 { - for _, it := range m { - p1, p2 := it[0], it[1] - t := &analysis.Token{ - Start: token.Start + p1, - End: token.Start + p2, - Term: token.Term[p1:p2], - Position: token.Position, - Type: analysis.AlphaNumeric, - } - ret = append(ret, t) +func (codeTokenizer) Tokenize(input []byte) analysis.TokenStream { + lower := bytes.ToLower(input) + seen := make(map[string]bool) + var tokens analysis.TokenStream + var starts [3]int + for end, count := 0, 0; end < len(lower); { + starts[0], starts[1], starts[2] = starts[1], starts[2], end + count = min(count+1, len(starts)) + _, size := utf8.DecodeRune(lower[end:]) + end += size + for _, start := range starts[len(starts)-count:] { + if gram := lower[start:end]; !seen[string(gram)] { + seen[string(gram)] = true + tokens = append(tokens, &analysis.Token{Start: start, End: end, Term: gram, Position: len(tokens) + 1, Type: analysis.AlphaNumeric}) } } } - return ret + return tokens } -func codeTokenFilterConstructor(_ map[string]any, _ *registry.Cache) (analysis.TokenFilter, error) { - return &codeTokenFilter{ - re: regexp.MustCompile("[a-zA-Z]+|[0-9]+"), - }, nil +func codeTokenizerConstructor(_config map[string]any, _cache *registry.Cache) (analysis.Tokenizer, error) { + return codeTokenizer{}, nil } diff --git a/modules/indexer/code/elasticsearch/elasticsearch.go b/modules/indexer/code/elasticsearch/elasticsearch.go index 74d749cd1d7..1e336ced05a 100644 --- a/modules/indexer/code/elasticsearch/elasticsearch.go +++ b/modules/indexer/code/elasticsearch/elasticsearch.go @@ -9,6 +9,7 @@ import ( "io" "strconv" "strings" + "unicode/utf8" repo_model "gitea.dev/models/repo" "gitea.dev/modules/analyze" @@ -23,12 +24,11 @@ import ( "gitea.dev/modules/setting" "gitea.dev/modules/timeutil" "gitea.dev/modules/typesniffer" - "gitea.dev/modules/util" "github.com/go-enry/go-enry/v2" ) -const esRepoIndexerLatestVersion = 3 +const esRepoIndexerLatestVersion = 4 var _ internal.Indexer = &Indexer{} @@ -55,6 +55,10 @@ const ( "tokenizer": "content_tokenizer", "filter" : ["lowercase"] }, + "content_short_analyzer": { + "tokenizer": "content_short_tokenizer", + "filter" : ["lowercase"] + }, "filename_path_analyzer": { "tokenizer": "path_tokenizer" }, @@ -64,8 +68,14 @@ const ( }, "tokenizer": { "content_tokenizer": { - "type": "simple_pattern_split", - "pattern": "[^a-zA-Z0-9]" + "type": "ngram", + "min_gram": 3, + "max_gram": 3 + }, + "content_short_tokenizer": { + "type": "ngram", + "min_gram": 1, + "max_gram": 2 }, "path_tokenizer": { "type": "path_hierarchy", @@ -87,7 +97,6 @@ const ( }, "filename": { "type": "text", - "term_vector": "with_positions_offsets", "index": true, "fields": { "path": { @@ -102,9 +111,16 @@ const ( }, "content": { "type": "text", - "term_vector": "with_positions_offsets", "index": true, - "analyzer": "content_analyzer" + "analyzer": "content_analyzer", + "fields": { + "short": { + "type": "text", + "analyzer": "content_short_analyzer", + "index_options": "docs", + "norms": false + } + } }, "commit_id": { "type": "keyword", @@ -236,23 +252,7 @@ func (b *Indexer) doDelete(ctx context.Context, repoID int64) error { return b.DeleteByQuery(ctx, es.TermsQuery("repo_id", repoID)) } -// contentMatchIndexPos find words positions for start and the following end on content. It will -// return the beginning position of the first start and the ending position of the -// first end following the start string. -// If not found any of the positions, it will return -1, -1. -func contentMatchIndexPos(content, start, end string) (int, int) { - startIdx := strings.Index(content, start) - if startIdx < 0 { - return -1, -1 - } - endIdx := strings.Index(content[startIdx+len(start):], end) - if endIdx < 0 { - return -1, -1 - } - return startIdx, (startIdx + len(start) + endIdx + len(end)) - 9 // remove the length since we give Content the original data -} - -func convertResult(searchResult *es.SearchResponse, kw string, pageSize int) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) { +func convertResult(searchResult *es.SearchResponse, terms []string, pageSize int) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) { hits := make([]*internal.SearchResult, 0, pageSize) for _, hit := range searchResult.Hits { repoID, fileName := internal.ParseIndexerID(hit.ID) @@ -270,22 +270,9 @@ func convertResult(searchResult *es.SearchResponse, kw string, pageSize int) (in return 0, nil, nil, fmt.Errorf("unexpected field types in search hit %q", hit.ID) } - // FIXME: There is no way to get the position the keyword on the content currently on the same request. - // So we get it from content, this may made the query slower. See - // https://discuss.elastic.co/t/fetching-position-of-keyword-in-matched-document/94291 - var startIndex, endIndex int - if c, ok := hit.Highlight["filename"]; ok && len(c) > 0 { + startIndex, endIndex := internal.ContentMatchIndexPos(content, terms) + if startIndex < 0 { startIndex, endIndex = internal.FilenameMatchIndexPos(content) - } else if c, ok := hit.Highlight["content"]; ok && len(c) > 0 { - // FIXME: Since the highlighting content will include and for the keywords, - // now we should find the positions. But how to avoid html content which contains the - // and tags? If elastic search has handled that? - startIndex, endIndex = contentMatchIndexPos(c[0], "", "") - if startIndex == -1 { - panic(fmt.Sprintf("1===%s,,,%#v,,,%s", kw, hit.Highlight, c[0])) - } - } else { - panic(fmt.Sprintf("2===%#v", hit.Highlight)) } hits = append(hits, &internal.SearchResult{ @@ -328,11 +315,19 @@ func extractAggs(searchResult *es.SearchResponse) []*internal.SearchResultLangua // Search searches for codes and language stats by given conditions. func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) { - searchMode := util.IfZero(opts.SearchMode, b.SupportedSearchModes()[0].ModeValue) - contentQuery := es.Query(es.NewMultiMatchQuery(opts.Keyword, "content").Type(es.MultiMatchTypeBestFields).Operator("and")) - if searchMode == indexer.SearchModeExact { - contentQuery = es.MatchPhraseQuery("content", opts.Keyword) + terms := internal.SearchTerms(opts) + if len(terms) == 0 { + return 0, nil, nil, nil } + contentQuery := es.NewBoolQuery() + for _, term := range terms { + if utf8.RuneCountInString(term) < 3 { + contentQuery.Must(es.NewMultiMatchQuery(term, "content.short").Operator("and")) + } else { + contentQuery.Must(es.MatchPhraseQuery("content", term)) + } + } + kwQuery := es.NewBoolQuery().Should( contentQuery, es.NewMultiMatchQuery(opts.Keyword, "filename^10").Type(es.MultiMatchTypePhrasePrefix), @@ -343,7 +338,6 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int } start, pageSize := opts.GetSkipTake() - kw := "" + opts.Keyword + "" languageAggs := map[string]any{ "language": map[string]any{ "terms": map[string]any{ @@ -353,15 +347,6 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int }, }, } - // number_of_fragments=0 returns the full highlighted content (no fragmentation). - highlight := map[string]any{ - "fields": map[string]any{ - "content": map[string]any{}, - "filename": map[string]any{}, - }, - "number_of_fragments": 0, - "type": "fvh", - } sort := []es.SortField{ {Field: "_score", Desc: true}, {Field: "updated_at", Desc: false}, @@ -375,12 +360,11 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int Size: pageSize, TrackTotal: true, Aggregations: languageAggs, - Highlight: highlight, }) if err != nil { return 0, nil, nil, err } - return convertResult(resp, kw, pageSize) + return convertResult(resp, terms, pageSize) } countResp, err := b.Indexer.Search(ctx, es.SearchRequest{ @@ -400,12 +384,11 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int From: start, Size: pageSize, TrackTotal: true, - Highlight: highlight, }) if err != nil { return 0, nil, nil, err } - total, hits, _, err := convertResult(resp, kw, pageSize) + total, hits, _, err := convertResult(resp, terms, pageSize) return total, hits, extractAggs(countResp), err } diff --git a/modules/indexer/code/elasticsearch/elasticsearch_test.go b/modules/indexer/code/elasticsearch/elasticsearch_test.go deleted file mode 100644 index e8f1f202ce0..00000000000 --- a/modules/indexer/code/elasticsearch/elasticsearch_test.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package elasticsearch - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestIndexPos(t *testing.T) { - startIdx, endIdx := contentMatchIndexPos("test index start and end", "start", "end") - assert.Equal(t, 11, startIdx) - assert.Equal(t, 15, endIdx) -} diff --git a/modules/indexer/code/indexer_test.go b/modules/indexer/code/indexer_test.go index 3a5cad97d76..1ad87e57773 100644 --- a/modules/indexer/code/indexer_test.go +++ b/modules/indexer/code/indexer_test.go @@ -5,18 +5,19 @@ package code import ( "context" - "os" + "fmt" "slices" "testing" "time" "gitea.dev/models/db" + repo_model "gitea.dev/models/repo" "gitea.dev/models/unittest" + "gitea.dev/modules/git" indexer_module "gitea.dev/modules/indexer" "gitea.dev/modules/indexer/code/bleve" "gitea.dev/modules/indexer/code/elasticsearch" "gitea.dev/modules/indexer/code/internal" - "gitea.dev/modules/setting" "gitea.dev/modules/test" "gitea.dev/modules/util" @@ -37,8 +38,29 @@ func TestMain(m *testing.M) { unittest.MainTest(m) } +const unicodeSearchContent = `Поставщик +日本語のテキスト +हिन्दी +ภาษาไทย +İSTANBUL +literal +abc bcd cde def +` + func testIndexer(name string, t *testing.T, indexer internal.Indexer) { t.Run(name, func(t *testing.T) { + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 62}) + require.NoError(t, git.ForceFastImport(t.Context(), repo, []git.FastImportCommit{{ + Ref: "refs/heads/master", + Files: []git.FastImportFile{ + {Path: "avocado.md", Content: "# repo1\n\npineaple pie of cucumber juice"}, + {Path: "cucumber.md", Content: "Salad is good for your health"}, + {Path: "example-file.js", Content: "console.log(\"Hello, World!\")"}, + {Path: "ham.md", Content: "This is also not cheese"}, + {Path: "potato/ham.md", Content: "This is not cheese"}, + {Path: "unicode.txt", Content: unicodeSearchContent}, + }, + }})) assert.NoError(t, setupRepositoryIndexes(t.Context(), indexer)) // Wait for the index to catch up: ES/OpenSearch make writes visible // only after a refresh (default interval: 1s). Bleve is synchronous @@ -57,6 +79,7 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) { Langs int SearchMode indexer_module.SearchModeType Results []codeSearchResult + Match string }{ // Search for an exact match on the contents of a file // This scenario yields a single result (the file README.md on the repo '1') @@ -89,6 +112,10 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) { Filename: "README.md", Content: "# repo1\n\nDescription for repo1", }, + { + Filename: "avocado.md", + Content: "# repo1\n\npineaple pie of cucumber juice", + }, }, }, // Search for an exact match on the contents of a file within the repo '2'. @@ -191,7 +218,7 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) { }, }, // Search for matches on the contents of files within the repo '62'. - // This scenario yields two results (both are based on contents, the first one is an exact match where as the second is a 'fuzzy' one) + // This scenario yields two results (both are based on contents, the first one is an exact match where as the second is a words match) { RepoIDs: []int64{62}, Keyword: "This is not cheese", @@ -209,10 +236,9 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) { }, // Search for matches on the contents of files regardless of case. { - RepoIDs: nil, - Keyword: "dESCRIPTION", - Langs: 1, - SearchMode: indexer_module.SearchModeFuzzy, + RepoIDs: nil, + Keyword: "dESCRIPTION", + Langs: 1, Results: []codeSearchResult{ { Filename: "README.md", @@ -257,6 +283,13 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) { }, }, }, + {RepoIDs: []int64{62}, Keyword: "ставщ", Langs: 1, SearchMode: indexer_module.SearchModeExact, Results: []codeSearchResult{{Filename: "unicode.txt", Content: unicodeSearchContent}}}, + {RepoIDs: []int64{62}, Keyword: "テキスト", Langs: 1, Results: []codeSearchResult{{Filename: "unicode.txt", Content: unicodeSearchContent}}}, + {RepoIDs: []int64{62}, Keyword: "हिन्दी", Langs: 1, SearchMode: indexer_module.SearchModeExact, Results: []codeSearchResult{{Filename: "unicode.txt", Content: unicodeSearchContent}}}, + {RepoIDs: []int64{62}, Keyword: "iSTANBUL", Langs: 1, SearchMode: indexer_module.SearchModeExact, Match: "İSTANBUL", Results: []codeSearchResult{{Filename: "unicode.txt", Content: unicodeSearchContent}}}, + {RepoIDs: []int64{62}, Keyword: "literal", Langs: 1, Match: "literal", Results: []codeSearchResult{{Filename: "unicode.txt", Content: unicodeSearchContent}}}, + {RepoIDs: []int64{62}, Keyword: "日本 ภาษา", Langs: 1, Match: "日本", Results: []codeSearchResult{{Filename: "unicode.txt", Content: unicodeSearchContent}}}, + {RepoIDs: []int64{62}, Keyword: "abcdef", SearchMode: indexer_module.SearchModeExact}, } for _, kw := range keywords { @@ -272,14 +305,15 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) { }) require.NoError(t, err) require.Len(t, langs, kw.Langs) + require.EqualValues(t, len(kw.Results), total) + require.Len(t, res, len(kw.Results)) hits := make([]codeSearchResult, 0, len(res)) - if total > 0 { - assert.NotEmpty(t, kw.Results, "The given scenario does not provide any expected results") - } - for _, hit := range res { + if kw.Match != "" { + require.Equal(t, kw.Match, hit.Content[hit.StartIndex:hit.EndIndex]) + } hits = append(hits, codeSearchResult{ Filename: hit.Filename, Content: hit.Content, @@ -307,7 +341,6 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) { func TestBleveIndexAndSearch(t *testing.T) { unittest.PrepareTestEnv(t) - defer test.MockVariableValue(&setting.Indexer.TypeBleveMaxFuzzniess, 2)() dir := t.TempDir() idx := bleve.NewIndexer(dir) @@ -322,13 +355,9 @@ func TestBleveIndexAndSearch(t *testing.T) { func TestESIndexAndSearch(t *testing.T) { unittest.PrepareTestEnv(t) - u := os.Getenv("TEST_INDEXER_CODE_ES_URL") - if u == "" { - t.SkipNow() - return - } + u := test.ExternalServiceHTTP(t, "TEST_ELASTICSEARCH_URL", "http://elasticsearch:9200") - indexer := elasticsearch.NewIndexer(u, "gitea_codes") + indexer := elasticsearch.NewIndexer(u, fmt.Sprintf("test_code_indexer_%d", time.Now().UnixNano())) if _, err := indexer.Init(t.Context()); err != nil { if indexer != nil { indexer.Close() diff --git a/modules/indexer/code/internal/util.go b/modules/indexer/code/internal/util.go index 3608dc4f12c..b55d9eefbd5 100644 --- a/modules/indexer/code/internal/util.go +++ b/modules/indexer/code/internal/util.go @@ -5,7 +5,10 @@ package internal import ( "strings" + "unicode" + "unicode/utf8" + "gitea.dev/modules/indexer" "gitea.dev/modules/indexer/internal" "gitea.dev/modules/log" ) @@ -46,3 +49,43 @@ func FilenameMatchIndexPos(content string) (int, int) { } return 0, len(content) } + +func SearchTerms(opts *SearchOptions) []string { + if opts.Keyword == "" { + return nil + } + if opts.SearchMode == "" || opts.SearchMode == indexer.SearchModeExact { + return []string{opts.Keyword} + } else { + return strings.Fields(opts.Keyword) + } +} + +// ContentMatchIndexPos returns the byte range of the earliest term in content, or -1, -1 unless all terms occur +func ContentMatchIndexPos(content string, terms []string) (int, int) { + lowerContent := strings.ToLower(content) + start, end := -1, -1 + for _, term := range terms { + lowerTerm := strings.ToLower(term) + position := strings.Index(lowerContent, lowerTerm) + if position < 0 { + return -1, -1 + } + if start < 0 || position < start { + start, end = position, position+len(lowerTerm) + } + } + lowerOffset, originalStart := 0, -1 + for offset := 0; offset < len(content); { + character, size := utf8.DecodeRuneInString(content[offset:]) + if lowerOffset == start { + originalStart = offset + } + lowerOffset += utf8.RuneLen(unicode.ToLower(character)) + offset += size + if lowerOffset == end { + return originalStart, offset + } + } + return -1, -1 +} diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/description b/tests/gitea-repositories-meta/org42/search-by-path.git/description deleted file mode 100644 index ffc40a9c48a..00000000000 --- a/tests/gitea-repositories-meta/org42/search-by-path.git/description +++ /dev/null @@ -1,9 +0,0 @@ -This repository will be used to test code search. The snippet below shows its directory structure - -. -├── avocado.md -├── cucumber.md -├── ham.md -├── potato -| └── ham.md -└── example-file.js \ No newline at end of file diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/post-receive b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/post-receive deleted file mode 100755 index 4b3d452abcc..00000000000 --- a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/post-receive +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -ORI_DIR=`pwd` -SHELL_FOLDER=$(cd "$(dirname "$0")";pwd) -cd "$ORI_DIR" -for i in `ls "$SHELL_FOLDER/post-receive.d"`; do - sh "$SHELL_FOLDER/post-receive.d/$i" -done \ No newline at end of file diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/post-receive.d/gitea b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/post-receive.d/gitea deleted file mode 100755 index 43a948da3a9..00000000000 --- a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/post-receive.d/gitea +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env bash -"$GITEA_ROOT/gitea" hook --config="$GITEA_ROOT/$GITEA_CONF" post-receive diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/pre-receive b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/pre-receive deleted file mode 100755 index 41270130536..00000000000 --- a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/pre-receive +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -ORI_DIR=`pwd` -SHELL_FOLDER=$(cd "$(dirname "$0")";pwd) -cd "$ORI_DIR" -for i in `ls "$SHELL_FOLDER/pre-receive.d"`; do - sh "$SHELL_FOLDER/pre-receive.d/$i" -done \ No newline at end of file diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/pre-receive.d/gitea b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/pre-receive.d/gitea deleted file mode 100755 index 49d09406364..00000000000 --- a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/pre-receive.d/gitea +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env bash -"$GITEA_ROOT/gitea" hook --config="$GITEA_ROOT/$GITEA_CONF" pre-receive diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/proc-receive b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/proc-receive deleted file mode 100755 index af2808b0370..00000000000 --- a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/proc-receive +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -ORI_DIR=`pwd` -SHELL_FOLDER=$(cd "$(dirname "$0")";pwd) -cd "$ORI_DIR" -for i in `ls "$SHELL_FOLDER/proc-receive.d"`; do - sh "$SHELL_FOLDER/proc-receive.d/$i" -done diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/proc-receive.d/gitea b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/proc-receive.d/gitea deleted file mode 100755 index 97521c62115..00000000000 --- a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/proc-receive.d/gitea +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env bash -"$GITEA_ROOT/gitea" hook --config="$GITEA_ROOT/$GITEA_CONF" proc-receive diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/update b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/update deleted file mode 100755 index c186fe4a18b..00000000000 --- a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/update +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -ORI_DIR=`pwd` -SHELL_FOLDER=$(cd "$(dirname "$0")";pwd) -cd "$ORI_DIR" -for i in `ls "$SHELL_FOLDER/update.d"`; do - sh "$SHELL_FOLDER/update.d/$i" $1 $2 $3 -done \ No newline at end of file diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/update.d/gitea b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/update.d/gitea deleted file mode 100755 index 38101c24266..00000000000 --- a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/update.d/gitea +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env bash -"$GITEA_ROOT/gitea" hook --config="$GITEA_ROOT/$GITEA_CONF" update $1 $2 $3 diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/info/refs b/tests/gitea-repositories-meta/org42/search-by-path.git/info/refs deleted file mode 100644 index 4adf83dda31..00000000000 --- a/tests/gitea-repositories-meta/org42/search-by-path.git/info/refs +++ /dev/null @@ -1,13 +0,0 @@ -90c1019714259b24fb81711d4416ac0f18667dfa refs/heads/DefaultBranch -985f0301dba5e7b34be866819cd15ad3d8f508ee refs/heads/branch2 -65f1bf27bc3bf70f64657658635e66094edbcb4d refs/heads/develop -65f1bf27bc3bf70f64657658635e66094edbcb4d refs/heads/feature/1 -78fb907e3a3309eae4fe8fef030874cebbf1cd5e refs/heads/home-md-img-check -9f894b61946fd2f7b8b9d8e370e4d62f915522f5 refs/heads/master -62fb502a7172d4453f0322a2cc85bddffa57f07a refs/heads/pr-to-update -4649299398e4d39a5c09eb4f534df6f1e1eb87cc refs/heads/sub-home-md-img-check -3fa2f829675543ecfc16b2891aebe8bf0608a8f4 refs/notes/commits -4a357436d925b5c974181ff12a994538ddc5a269 refs/pull/2/head -5f22f7d0d95d614d25a5b68592adb345a4b5c7fd refs/pull/3/head -62fb502a7172d4453f0322a2cc85bddffa57f07a refs/pull/5/head -65f1bf27bc3bf70f64657658635e66094edbcb4d refs/tags/v1.1 diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/objects/.keep b/tests/gitea-repositories-meta/org42/search-by-path.git/objects/.keep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/objects/info/packs b/tests/gitea-repositories-meta/org42/search-by-path.git/objects/info/packs deleted file mode 100644 index 9774923d2ea..00000000000 --- a/tests/gitea-repositories-meta/org42/search-by-path.git/objects/info/packs +++ /dev/null @@ -1,2 +0,0 @@ -P pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.pack - diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.bitmap b/tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.bitmap deleted file mode 100644 index 39c02c29877a8809da1e67f7cb474efab663d5cc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 678 zcmZ?r4Dn@PWME}rVBlW9?|aU-M_aN_h)vGgHks|Wz`CV{AaNjOf?^;A0_^`GG=n&h zMwP4o0hMAa;VhHT!*Z-#h-y;iUR{f&D&QS9G0B;)$#VY;i3t~Gku?h G76SlJfIs~J diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.idx b/tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.idx deleted file mode 100644 index 38d0e6b72265f254a87cf9777741ddcead7f60da..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2108 zcmb`IYc$k(7{~w2#f+Ab+=kpql(8A)S|N^0L$j3JA~dyBZj*9bsn}zrb!kgw4I0!g zV?$HyDr?J_n2s7-ggO(;FxgC7&ARSBUO30;9B0|>#pnFL-{6hI~Gv@y@(>{MF^ljMhR;F zg|#rZ6lL(su@2r=iz>`4MGgE?)WI)@2D~jo1ichZ@LDTiJ@hP~y$W=occth;?`qJ4 z{zddbe~bat441(O>i-90m|q>o6$j*=I)`@2u~F19yHr_&+s!yCN1J)BY;43vT}YHG zJ}=5$U!|OhVJFhAw6Qu}j;q>Ewukt?VyH!0+6v-W&Qp00{YWkstBChI!ptnR-QyA% zBqemIRq(TlB=_8L&bAxsz5Z4jUM=^9NhB6i8(+e3*k19M>7hNYuV>Zjb2P=Gdsy7{ zcRQSt+R>eRiwv`Uuw|)PJ+wj(S$9v)}p7rCE? z_v!DXL|?14-MjXII47HTQM1JAhGTQ>%*mPjuk6YA@!|Pz-k@;wM?XpU`_m{AReutO zi@d!=EzOU8>R3{YKJ$!FKAGi?vM0%&GcPMW7sYSsagZC9f6V-|B$?6TXiy3TBtz<`Q8ERGG zi4L9oCPtckeK6YWnCa2$;w#kwt>Y0ok==8#zxtIQpW_lsL$!a*#$Fa59^@nQQbRUx zov*o9kWqd3zJ&Ng8ZLEfP{({nTzNgg>4Poz08r}=%UDr5J{)0*kRtd>X`=4MGI7hj!cgl6sttclHL z)qF1MAFyw<+v&QMn@6HMO>|}LK-SfG&E5{%_d+-<7;&9T0nF5W|}wI_=<_iXbXqfQForJq2;u4QCm0L*WGI z8U&Gdg3}7O57h)G;0{D|A-s!V2C`bE0si@X*k=v(Aq2rrfrtB|@*LvS6o`fR_kb^C z>DqaypN3wzGs*(ct}p|6v=HCQtm?bcBOhKh#&oC^MRSU|iMS4hS2@#M+7(z}3Qi=co0QKntXW@EfhmT9(_CbN-)BY-)6lzQe9z+mn zE3L({66ZI4n8wL9xp2@55%Q`6m!u>pSg$m}C6{}%K+(6NLqDQU%u_g@dOR0%zTwz8)x_4AqM!j7O zIJo?D1QJn3Pg$rJ3sN+1E$&Y}&Y;}klV`VZK>dUUhsw|*(j`LY4PDCE{r|d(+M?@My$7e_*0nzy0AD%N|W2Am^D^c5%Ff zT|X2kb0s)vm}mAokrhY5mVu$tL)|C1o+n!Q1M_#T{MK6XZizIahrl}%{%3m;!DX)~ z5HIY1fRW~=Qr&xNM8a;Od+SOXo@6y!MrV3_<~6(Lt#Io@>CW}%8@ zXuN=5|AoywjL&P5K($rOuRCqqx_XTHM3KroN0tk71Bajh*V1TLe_M?30_+=JF1&tz z<`MAa_cxURrVQO41YtaktH@;w+Z8k7{<)@}zW6T9&1j(atW~Jjq<=wPsdgT}8ex4A zD~(F`9XQ*0ypsL!h2?_N^sj~FpT_={$07=_kBCAR-oJJI+jARFp6g^sNAP?JiF$QH z2(v1{uuw<}SzB{>GF|+YtOce^qS7B`xkJkZod%a_D;H*=9IINrOp*c$GphJy)5BHZ zWN?`?(reojatMzvPsPj2>29D@@0Xwi57tv|iwV#zY@hC|I zK&eDt*`LNIKJ-JMiql!T-W|oDQ4kQ6C-1R`+5{;?u@guD@%&nR`RAd>x{z0GI|)_0 zbtt`zzNTqFjbbAVGB3d1Q(n5yT<9K3pR0MIj(6I3(Uk6NlvtG9D_W^T9&MSF=8r@( zmZQQh5L|X%ukN<=eQ0VwrZTjAUTl!!&@kS;^opM1EZ1-RWaC@={_mDZJt`;U?j}Xg zq-|^WqoS&(wiop+*qWUp`{=_g-OjeWL$1`5W;kr>Ocof1d?0g`!!aI?P3Dj0uOnE~ zZTqDAzAH#~yq3tsWrD$fPf8KoTyc;QA&dpIz;qoC4kb--@pci%N`HaQ9IFvA7zHDV zZG6Fy>owAeZaMG* zCfekB;hXx11BsL4fk_So)4&oL6E8m)C~H1El5ICqR}$Y{+DmU2W%PVMrvSn~CZ~_bnK6N5oe)b+U&P<%{|!y=(RSq#}SzU=Wu!r@qZ%Ia-qN z%||MzsTlIxIX>%UHdwdAF$YN|zcf=Pitr^h!fR*Sg%+;MK)^j7dKuETmd@S+-cS>q zbt{?k_QXrY)av2jRJtSK>sZQKELq>KXMZ?^suFEt33J-|cTOP^!48pX^TT60S!^f@ zAlaAYTJxH0grrKfD0|To;KbCjT)k&~T=MctA zZzH{ncSyfne^sScBEv5VTL9`U+KqX3`zNSVoQ`Dms!>7F!+R1GNxo_N{tjcU zas_^?KH9)lX2PGI9_M`BuA%;9VYxJ7k^xYR;~fJe6D$3>#3(06h$Fe(qK8(BKiw}; z(!fBbDEnA}K*s{0yYe~k@2LGcE3}B1%C{$KhU)&qm2bOpZ8|f~y8GAXzM3FfCmzq0 zVp&y!B`4z$)6@Z;1h<#2oODP*_wv$2_vos?{mPeCLuJ-GN`MOeW6s~{1Gu55P?t6yndX_pN-p45h6ghf+m#ZEmqGOFl_?v}<>^}GIGry3?bSC@V-AI7gg~Mr7 zpsj||&6l??oKl0!;(P1uTs9rgzpEdoNdGH8h?+`Sfb^B03{rj53X_>bfvPT}78JX2 z{G2itv}^@kubuKm@Wsp(kb||^6a>$sIMuwuEjjyNOEM^Mu_%D8o`;VW8}2k}Oy;qY zS7|ZNaZ{Z_(ei^(S=$X+m$`wm65+$lx*UNnf)QGeU|REPLS3qEM%T#R6_j z7>6ys@O(+TR_(=3>Vu#rlZs1WN>MHNPgR*9;I_ChfKS9pz(m|8SA^6E z?f!E%Dyfa1qSy;Jt!MfwpK8wI>#{cawDyLPk=dUI2;-yrjtdXf3;L)%zpBCQM%E_ zlkUyia4jVm(lUCQ4}1`c#hIMyoG}`l(^9UqAgC^TSxar6sv~z%u$1!lb>Vh^(7X!D zLfH7Qo1eZm5>^y15OxbkIO`iA!S%xzQb!o6nE5!lNU0h#(FqGsb0(xc4_KU}=IvRU%?caLO(B|c{+ zsdOs&mqxrimd8t=q`hnl)O){rtr`R@hF3c?U2S3I#eP>md*@Pml zQ9GzHS}d>XhSE{d76-yEK4BciXUx5)o*eRKc!jKsFz zOU*d4VUwUP+N~SY@vp-lTdi-LRJS80jr^vN3Vp~3uuAOr16qt!Nbm}!Klp-bV`+q@ z!*b(ID?xRbhJ;l)k3*!aRO_#P)rH+YLRE8l8X=z+AFpB*j5o&WqWJ`#Gu-_?>O(`K ztI2uK+?U^;Cg5}DhR+`5A{;;8biA@GfE`R<&d_{R4Nn&P+-3IOO53VNEI@5tuCNE^ zfvQvS;j|hOfZh&QmNV?00LMnXDD&U%p!SA?rwqCJ4{jw_PYY&Ncp5F> z>I2y5ff+pdz-0>by~Cf~9tXWNZcY_Khq=S7Xi+D&lhBafMi41>2-KRB_<6P&WT$-O znNl6PcSZbgRZcKuODU-l!eE&6GCigXLa&}6tuiA0%v@BFBZV1Nr;4N7daHj=qRry()OiZUR67c@R}3_>>f2m3D{r> zm{n(@7MZ2L0aj$aTB)dA6FXuasegFd{>Zz3bOnzA>oPCfp1k>H^XdOd|so#))70zq1dATQ# zGWi)DY`LJyxmZh2QCoUAH^h4CC)LbX6~5*f>Faa{HXTxZk;C&xM|Zhq$buNu=$#kT z%nsz?3bv+0M-@*l;~EV)H#V^_OT6zOF}|E@61!QlnmC=DV>{WO z3)U{4PrT&kE&FqY6FFDgKYHG@GVnUM+UpUAHQ!R>`3CZyIi(v8rFnxiOtqvI5gsci z;h<_xsS!cKoDXK?2@3L6ecySU;Esafwyx#Ddzqps-R2bW5m#t>&deuqe_gL5k__>~ z$y(PuyiMk^eCjqocvB5U5Y;pMX5B4jQ6?q48f#R{y_Usd%n?w3#tb#GmH9}>Q*dwN zLI4gB@fi87upyj$?vUDOD^q2MBO=t6i{vJ{(O;vGV9GJ*-aEfyiWjc1C7l{mW;17; zV>>wR%;Mc6OgxNX9u=7tPP3f|)2wvqBhR(l+GchfwhG&?D+JEe1{`tq9mz6 z5H*?g9Vs|3>W*e>zt&$!zbc$YBgXGSIIt5uXkr|KgTu(_i-$MGum zu=Ayr+WGcvxB*q9=KAt>nfXm>X2F(KWpAgU8UfEbl+Jk+x3d+8HvaDQexFrA{|OgH z+Wd?W+s|HIp}J2MIetV{SLbD{?1%*r{oGhP>ZZnAMk(}0Gokl!HFuHqa1Vw-6D=9f78;ZSH@%2BBH5v$FKR%@`kn+3M9Tg3Cp| z;QJ8CHy!$^2A@{Ru)P~-L#U^{i$EN#t4;-z8zaELFt;g3cK3$a(||x>{jY68*($rU zo5tu6WOgLY!n|y5Dfj+M?*>}kmAA|CVlMREmXlu%e??A^Bso|b&%H;<*URcD;pMPq zixGW6ZADl?a%ub7G3A)ta*rjc5ULYF9c63`9FF58DRn@_Ub)~}(}BGpDc&B;&kU{h zq|5*D8UNlv2Xr@DZI#o~lC$vlu*%gm(1XYQ4JMVX*8N{sq*U^OeP z`#JfIlg3g<;yF)Fywk>?eB9gN`-;g_CtRpv>UsRYQJQ(&-YIB3HmyD z$~RnSoN;?^$=V93`WXn}Zgkn|e4F}w*q^b8ig$8z!zIybwdZo&^FxLJ?VwDK50Q4z zYee}wT+J4GCI-THB|Z0D=2PAI1|^r~!|z<-Zx*ME2~v%A1*Ubs+4q46DbiY*m+R@M zuQ{tl>91BmV+M~C&*Qp8E~N(Uu}WQ(UMD}D8{Nm13H#`Q;u|8R>3BAPQnjQZw_kl; z(dDsh^=eY3ABV3TkIsB!UZsFom$E+YbzQcA$7JpC{d%pUa;rS|mV<4bqWLmXaaTs? z#pO74jCE(v;b#^*_wEQe==X?kZ~mZoP&VrzGvU~}ZxGTL?09KBlEZX}`^tdvPel!aFmZLIcim_%3emcC_ zY(P!GP?4kvS9FDI#E6e()}Gg%YTg3hYi4!Z`Fr6lPD%|~=Z|Wz5WC{JXt>XExoTWA zM1#mh0y?{{-xgo*G}&Gp3Kz3{Py(UlD{vFLN<@h89UIYt=Ro`RW%)>ho3H1wOLUqf zX!L535e^tD!uorsnBV1;-A+3*64X;Bx0FNJrICjQ&{vR)C_w#3^@#lV{FCB31LlC2 z1JZr=$izrXAH|(9m&yGGOhCIcK3N(1Kj4#7H;)TE4@;okJ0sm}zQb=xvns^}^93T|nBcr;tg~~f<0H+zsp<0wDK288?c=Fwr zdBoJ0Yl}G5$6x(TPQyL$t1hzWht(b;z`!RWqT>Ik#Qulasg6x($VKpdgG(0OO31+i zi!EF@9Wcy^&rtLqoltINZY2e7E>#6S6$Ng7d2Vh_ZeD)wW+*qGGA}oG9S=+yrl7(F zGiz&bTFOEC<_K=j`-$99I?x2B(!2lwlGn(Gt1$nWHq^=#vfbGqFu{3=e=E4to0dAS zq_E-%l4J}uq%ra{x^U5a-X(&%h`BaZkyGo_FvHkzHh*XXzfqIP7`*8qCof6ui0}cE zf$>R>4a0Lj6x@XP-r-)U?dHcv1`Wc@uEsIHaHzR{GM zB{21vagJb_ZuTT#unYO;v8j=CUMY-eejAPt8|&f4f$@ryF)0{;v~6mX|60DceC2mx zt6@1tyoOl?dQE3MuDV^h7=ZZ+0^R?Prwv%wPBJ_T|8f%7_CER`& z%;1$h=A5~X4nSK)3I4CQ{~tD(86#+FeOg^j9f$(k2>;gR>ydFqW!4dxHe^Iedqh=J zo?V%JLRo=LW90ROk_rak;ejBc`^h(WD&sIE^$}Jb`Kc*Yb@?$4jk0l$CzQPMI5PqO z$n3;T>w1DIPEN5*PQm8K5UPu6XYvyW#G*R(6vrq`U26s%0H?z-Z~Ye;;)w`QFp$0J z7v6P89KIDhH@B$;Ft5QjZ}`vf{~8f_i3x9$^vX8UP>flD0(*9K@db`TmQssLHE^S= z84Kbp-&xL5(-_2QQ?tBP=zqAKge*C<0->41SDEj_B3AjV{#fxT{-K* zD<&3rL&!pHbiQ|$rjdc;9K{~*E;XQW|d@bun=t~qZiTKQds-3 zWiqYb)c>=%J${KJ>8un-cFL*w8@cr*LDl}yM;V+ zPqUR)Ao0|XOq3ZiDZ$aoKb<59?M!MO`cw^;dpLYwQm;+7>PXVPdMwxxq~S=s2iU!; zTr=pYRlohZiY8?o+xjp(Dg8%ueS^&^&_i?U_U8`CS*?JUOSz|#GmtD`h*9o`-CF2;wnUER3tdHgxc&jXo58i{a zHt6%jQ7438FBwy+ni*cvCCM#GB*|M`Bj66P1Sna_HqpVgByY7sLx*~wJB&@bUTdU} zzw4XoNHp&{$90h1#F?;0JzIsYy2G z-$(W9SU4M~r`AMdwxCc_t%vg?j>l3dSuf3-UYip`f}1G8{s)OIk}dT3uoJgw^8ukZ z4kyI8)kSrx;GLA2H?jQ;nkZdh!))oVel~31vC!AT>gm`DJD{e)kik{OZzE@K!Yq#> zOd#Z%(XxFQ`h+^@Q)MK*omy}EccyCzMm(UiAR}>`QS682G0-leTZ(; z&Qo5vOgS0aNe=Ia-<@(K&SG;6wC+?l+dkl^#x9)tqrAggd;Fp8SsVM?>mBb?XP6?= z-9iPR52C=@{=dTF2<;R@7_Aw1ic3Re;L*hAN!+n*#KC%v)ehvGe}`>i=cC diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.rev b/tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.rev deleted file mode 100644 index b06ecca825a3caffbe0dd488445f64b677b583ce..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 200 zcmXZTyAHu%0EXdTQF=ff+M>>^ZbB?plWV}z3+Po;MC>9KArWI2i@EN=s5^tj+r*bV z`KEU|8P9|ei9ee!BMSpEbmU+`gMvI9G!eH+4A4Oj1(+zJixgTYp^84zkf0)iHsV## oKn?YM@L9f}m&M)k=T~3;8poZh9M-`j3bobdH9ViT(&%P-Klc<{9 diff --git a/tests/gitea-repositories-meta/user2/glob.git/objects/59/fee614e09d1f1cd1e15e4b2a7e9c8873a81498 b/tests/gitea-repositories-meta/user2/glob.git/objects/59/fee614e09d1f1cd1e15e4b2a7e9c8873a81498 deleted file mode 100644 index a55c8cc6e69bdfd2433025eb8961c2c9bc51ccd2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 34 qcmb!NCp${7X diff --git a/tests/gitea-repositories-meta/user2/glob.git/objects/7c/8ac2f8d82a1eb5f6aaece6629ff11015f91eb4 b/tests/gitea-repositories-meta/user2/glob.git/objects/7c/8ac2f8d82a1eb5f6aaece6629ff11015f91eb4 deleted file mode 100644 index d5176e68c637f3eba8b8711529832d0a4a9d9c12..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 21 ccmb6WH2-^Ff%bxNYpE-C}G%tZCQl|D8E6aR};0H1yxy10rWrT_o{ diff --git a/tests/gitea-repositories-meta/user2/glob.git/objects/95/aff026f99a9ab76fbd01decb63dd3dbc03e498 b/tests/gitea-repositories-meta/user2/glob.git/objects/95/aff026f99a9ab76fbd01decb63dd3dbc03e498 deleted file mode 100644 index 0883f2b3eb679bc51391d23bdffba8149f187cac..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 34 qcmb