diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml
index 56dc4733abe..db5845864e6 100644
--- a/.github/workflows/pull-db-tests.yml
+++ b/.github/workflows/pull-db-tests.yml
@@ -215,7 +215,6 @@ jobs:
run: GITEA_TEST_DATABASE=mysql make test-integration
env:
TAGS: bindata
- TEST_INDEXER_CODE_ES_URL: "http://elastic:changeme@elasticsearch:9200"
test-mssql:
if: needs.files-changed.outputs.backend == 'true'
diff --git a/modules/indexer/code/bleve/bleve.go b/modules/indexer/code/bleve/bleve.go
index d22f735571a..bcacef245b4 100644
--- a/modules/indexer/code/bleve/bleve.go
+++ b/modules/indexer/code/bleve/bleve.go
@@ -7,9 +7,11 @@ import (
"context"
"fmt"
"io"
+ "slices"
"strconv"
"strings"
"time"
+ "unicode/utf8"
repo_model "gitea.dev/models/repo"
"gitea.dev/modules/analyze"
@@ -68,7 +70,7 @@ const (
repoIndexerAnalyzer = "repoIndexerAnalyzer"
filenameIndexerAnalyzer = "filenameIndexerAnalyzer"
repoIndexerDocType = "repoIndexerDocType"
- repoIndexerLatestVersion = 11
+ repoIndexerLatestVersion = 12
)
// generateBleveIndexMapping generates a bleve index mapping for the repo indexer
@@ -80,6 +82,8 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) {
textFieldMapping := bleve.NewTextFieldMapping()
textFieldMapping.IncludeInAll = false
+ textFieldMapping.DocValues = false
+ textFieldMapping.IncludeTermVectors = false
docMapping.AddFieldMappingsAt("Content", textFieldMapping)
fileNamedMapping := bleve.NewTextFieldMapping()
@@ -105,7 +109,7 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) {
"type": analyzer_custom.Name,
"char_filters": []string{},
"tokenizer": codeTokenizerName,
- "token_filters": []string{unicodeNormalizeName, codeTokenFilterName, lowercase.Name},
+ "token_filters": []string{},
}); err != nil {
return nil, err
}
@@ -141,7 +145,6 @@ func (b *Indexer) SupportedSearchModes() []indexer.SearchMode {
func init() {
// due to bleve's design problem, the "Register" must be done in the main goroutine, otherwise data-race
util.MustNoError(registry.RegisterTokenizer(codeTokenizerName, codeTokenizerConstructor))
- util.MustNoError(registry.RegisterTokenFilter(codeTokenFilterName, codeTokenFilterConstructor))
util.MustNoError(registry.RegisterTokenizer(pathTokenizerName, pathTokenizerConstructor))
}
@@ -266,25 +269,15 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
pathQuery.FieldVal = "Filename"
pathQuery.SetBoost(10)
- searchMode := util.IfZero(opts.SearchMode, b.SupportedSearchModes()[0].ModeValue)
- if searchMode == indexer.SearchModeExact {
- // 1.21 used NewPrefixQuery, but it seems not working well, and later releases changed to NewMatchPhraseQuery
- q := bleve.NewMatchPhraseQuery(opts.Keyword)
- q.Analyzer = repoIndexerAnalyzer
- q.FieldVal = "Content"
- contentQuery = q
- } else /* words */ {
- q := bleve.NewMatchQuery(opts.Keyword)
- q.FieldVal = "Content"
- q.Analyzer = repoIndexerAnalyzer
- if searchMode == indexer.SearchModeFuzzy {
- // this logic doesn't seem right, it is only used to pass the test-case `Keyword: "dESCRIPTION"`, which doesn't seem to be a real-life use-case.
- q.Fuzziness = inner_bleve.GuessFuzzinessByKeyword(opts.Keyword)
- } else {
- q.Operator = query.MatchQueryOperatorAnd
- }
- contentQuery = q
+ terms := internal.SearchTerms(opts)
+ if len(terms) == 0 {
+ return 0, nil, nil, nil
}
+ contentQueries := make([]query.Query, 0, len(terms))
+ for _, term := range terms {
+ contentQueries = append(contentQueries, inner_bleve.MatchAndQuery(term, "Content", repoIndexerAnalyzer, 0))
+ }
+ contentQuery = bleve.NewConjunctionQuery(contentQueries...)
keywordQuery = bleve.NewDisjunctionQuery(contentQuery, pathQuery)
@@ -316,67 +309,79 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
}
from, pageSize := opts.GetSkipTake()
- searchRequest := bleve.NewSearchRequestOptions(indexerQuery, pageSize, from, false)
- searchRequest.Fields = []string{"Content", "Filename", "RepoID", "Language", "CommitID", "UpdatedAt"}
- searchRequest.IncludeLocations = true
-
- if len(opts.Language) == 0 {
- searchRequest.AddFacet("languages", bleve.NewFacetRequest("Language", 10))
+ // grams prove terms of up to 3 runes, longer ones are verified against the content, so fetch candidates until the page is full
+ matched, scanned := 0, 0
+ if !slices.ContainsFunc(terms, func(term string) bool { return utf8.RuneCountInString(term) > 3 }) {
+ matched, scanned = from, from
}
+ var result *bleve.SearchResult
+ var total int64
+ searchResults := make([]*internal.SearchResult, 0, pageSize)
+ for {
+ size := from + pageSize - matched
+ searchRequest := bleve.NewSearchRequestOptions(indexerQuery, size, scanned, false)
+ searchRequest.Fields = []string{"Content", "Filename", "RepoID", "Language", "CommitID", "UpdatedAt"}
+ searchRequest.IncludeLocations = true
+ if result == nil && len(opts.Language) == 0 {
+ searchRequest.AddFacet("languages", bleve.NewFacetRequest("Language", 10))
+ }
+ searchRequest.SortBy([]string{"-_score", "UpdatedAt"})
- searchRequest.SortBy([]string{"-_score", "UpdatedAt"})
+ batch, err := b.inner.Indexer.SearchInContext(ctx, searchRequest)
+ if err != nil {
+ return 0, nil, nil, err
+ }
+ if result == nil {
+ result = batch
+ }
- result, err := b.inner.Indexer.SearchInContext(ctx, searchRequest)
- if err != nil {
- return 0, nil, nil, err
+ for _, hit := range batch.Hits {
+ scanned++
+ content, okContent := hit.Fields["Content"].(string)
+ language, okLanguage := hit.Fields["Language"].(string)
+ commitID, okCommitID := hit.Fields["CommitID"].(string)
+ updatedAt, okUpdatedAt := hit.Fields["UpdatedAt"].(string)
+ repoID, okRepoID := hit.Fields["RepoID"].(float64)
+ if !okContent || !okLanguage || !okCommitID || !okUpdatedAt || !okRepoID {
+ hitFieldsJson, _ := json.Marshal(hit.Fields)
+ setting.PanicInDevOrTesting("unexpected field types in search hit %q: %s", hit.ID, string(hitFieldsJson))
+ return 0, nil, nil, fmt.Errorf("unexpected field types in search hit %q", hit.ID)
+ }
+
+ var startIndex, endIndex int
+ if len(hit.Locations["Filename"]) > 0 {
+ startIndex, endIndex = internal.FilenameMatchIndexPos(content)
+ } else if startIndex, endIndex = internal.ContentMatchIndexPos(content, terms); startIndex < 0 {
+ continue
+ }
+ matched++
+ if matched <= from {
+ continue
+ }
+
+ var updatedUnix timeutil.TimeStamp
+ if t, err := time.Parse(time.RFC3339, updatedAt); err == nil {
+ updatedUnix = timeutil.TimeStamp(t.Unix())
+ }
+ searchResults = append(searchResults, &internal.SearchResult{
+ RepoID: int64(repoID),
+ StartIndex: startIndex,
+ EndIndex: endIndex,
+ Filename: internal.FilenameOfIndexerID(hit.ID),
+ Content: content,
+ CommitID: commitID,
+ UpdatedUnix: updatedUnix,
+ Language: language,
+ Color: enry.GetColor(language),
+ })
+ }
+ if matched == from+pageSize || len(batch.Hits) < size {
+ total = max(int64(matched), int64(matched)+int64(batch.Total)-int64(scanned))
+ break
+ }
}
-
- total := int64(result.Total)
-
- searchResults := make([]*internal.SearchResult, len(result.Hits))
- for i, hit := range result.Hits {
- content, okContent := hit.Fields["Content"].(string)
- language, okLanguage := hit.Fields["Language"].(string)
- commitID, okCommitID := hit.Fields["CommitID"].(string)
- updatedAt, okUpdatedAt := hit.Fields["UpdatedAt"].(string)
- repoID, okRepoID := hit.Fields["RepoID"].(float64)
- if !okContent || !okLanguage || !okCommitID || !okUpdatedAt || !okRepoID {
- hitFieldsJson, _ := json.Marshal(hit.Fields)
- setting.PanicInDevOrTesting("unexpected field types in search hit %q: %s", hit.ID, string(hitFieldsJson))
- return 0, nil, nil, fmt.Errorf("unexpected field types in search hit %q", hit.ID)
- }
-
- startIndex, endIndex := -1, -1
- for _, locations := range hit.Locations["Content"] {
- location := locations[0]
- locationStart := int(location.Start)
- locationEnd := int(location.End)
- if startIndex < 0 || locationStart < startIndex {
- startIndex = locationStart
- }
- if endIndex < 0 || locationEnd > endIndex {
- endIndex = locationEnd
- }
- }
- if len(hit.Locations["Filename"]) > 0 {
- startIndex, endIndex = internal.FilenameMatchIndexPos(content)
- }
-
- var updatedUnix timeutil.TimeStamp
- if t, err := time.Parse(time.RFC3339, updatedAt); err == nil {
- updatedUnix = timeutil.TimeStamp(t.Unix())
- }
- searchResults[i] = &internal.SearchResult{
- RepoID: int64(repoID),
- StartIndex: startIndex,
- EndIndex: endIndex,
- Filename: internal.FilenameOfIndexerID(hit.ID),
- Content: content,
- CommitID: commitID,
- UpdatedUnix: updatedUnix,
- Language: language,
- Color: enry.GetColor(language),
- }
+ if total == 0 {
+ return 0, nil, nil, nil
}
searchResultLanguages := make([]*internal.SearchResultLanguages, 0, 10)
@@ -384,9 +389,9 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
// Use separate query to go get all language counts
facetRequest := bleve.NewSearchRequestOptions(facetQuery, 1, 0, false)
facetRequest.Fields = []string{"Content", "RepoID", "Language", "CommitID", "UpdatedAt"}
- facetRequest.IncludeLocations = true
facetRequest.AddFacet("languages", bleve.NewFacetRequest("Language", 10))
+ var err error
if result, err = b.inner.Indexer.Search(facetRequest); err != nil {
return 0, nil, nil, err
}
diff --git a/modules/indexer/code/bleve/bleve_test.go b/modules/indexer/code/bleve/bleve_test.go
index 0a59b677e13..06d55de64f4 100644
--- a/modules/indexer/code/bleve/bleve_test.go
+++ b/modules/indexer/code/bleve/bleve_test.go
@@ -4,12 +4,15 @@
package bleve
import (
+ "strconv"
"testing"
"time"
"gitea.dev/models/db"
+ indexer_module "gitea.dev/modules/indexer"
"gitea.dev/modules/indexer/code/internal"
inner_bleve "gitea.dev/modules/indexer/internal/bleve"
+ "gitea.dev/modules/util"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -32,7 +35,7 @@ func TestBleveIndexerTokenFilter(t *testing.T) {
expectedIDs []int64
}{
{keyword: "12345", expectedIDs: []int64{2}},
- {keyword: "DNS", expectedIDs: []int64{}},
+ {keyword: "DNS", expectedIDs: []int64{2}},
{keyword: "mdns", expectedIDs: []int64{2}},
{keyword: "port", expectedIDs: []int64{2}},
{keyword: "port2", expectedIDs: []int64{2}},
@@ -54,6 +57,38 @@ func TestBleveIndexerTokenFilter(t *testing.T) {
}
}
+func TestBleveIndexerPagesVerifiedCandidates(t *testing.T) {
+ indexer := NewIndexer(t.TempDir())
+ defer indexer.Close()
+ _, err := indexer.Init(t.Context())
+ require.NoError(t, err)
+
+ batch := inner_bleve.NewFlushingBatch(indexer.inner.Indexer, maxBatchSize)
+ for i, content := range []string{"abc bcd cde def", "abcdef", "abc bcd cde def", "abcdef", "abc bcd cde def"} {
+ filename := strconv.Itoa(i)
+ require.NoError(t, batch.Index(internal.FilenameIndexerID(1, filename), &RepoIndexerData{RepoID: 1, Filename: filename, Content: content, UpdatedAt: time.Now()}))
+ }
+ require.NoError(t, batch.Flush())
+
+ var filenames []string
+ for page := 1; page <= 3; page++ {
+ total, results, _, err := indexer.Search(t.Context(), &internal.SearchOptions{
+ Paginator: &db.ListOptions{Page: page, PageSize: 1},
+ Keyword: "abcdef",
+ SearchMode: indexer_module.SearchModeExact,
+ })
+ require.NoError(t, err)
+ require.Len(t, results, util.Iif(page < 3, 1, 0))
+ if page == 3 {
+ assert.EqualValues(t, 2, total)
+ }
+ for _, result := range results {
+ filenames = append(filenames, result.Filename)
+ }
+ }
+ assert.ElementsMatch(t, []string{"1", "3"}, filenames)
+}
+
func searchResultIDs(result []*internal.SearchResult) []int64 {
ids := make([]int64, 0, len(result))
for _, hit := range result {
diff --git a/modules/indexer/code/bleve/token_code.go b/modules/indexer/code/bleve/token_code.go
index 53a6c401a0e..6a7d6d6d08b 100644
--- a/modules/indexer/code/bleve/token_code.go
+++ b/modules/indexer/code/bleve/token_code.go
@@ -4,55 +4,38 @@
package bleve
import (
- "regexp"
- "unicode"
+ "bytes"
+ "unicode/utf8"
"github.com/blevesearch/bleve/v2/analysis"
- "github.com/blevesearch/bleve/v2/analysis/tokenizer/character"
"github.com/blevesearch/bleve/v2/registry"
)
const codeTokenizerName = "codeTokenizer"
-func codeTokenizerConstructor(_ map[string]any, _ *registry.Cache) (analysis.Tokenizer, error) {
- // Old code used "letter" tokenizer which doesn't support CJK.
- // Here it still doesn't support CJK, since there is no usable CJK tokenizer at the moment.
- return character.NewCharacterTokenizer(func(r rune) bool {
- return unicode.IsLetter(r) || unicode.IsNumber(r)
- }), nil
-}
+// codeTokenizer emits every distinct lowercase gram of 1 to 3 runes once, search verifies longer terms against the content
+type codeTokenizer struct{}
-const codeTokenFilterName = "codeTokenFilter"
-
-type codeTokenFilter struct {
- re *regexp.Regexp
-}
-
-func (c codeTokenFilter) Filter(stream analysis.TokenStream) (ret analysis.TokenStream) {
- // split one token to "letter" parts and "number" parts (to keep the old behavior).
- // e.g.: input token="port123", then the output tokens are "port123", "port", "123"
- for _, token := range stream {
- ret = append(ret, token)
- m := c.re.FindAllIndex(token.Term, -1)
- if len(m) > 1 {
- for _, it := range m {
- p1, p2 := it[0], it[1]
- t := &analysis.Token{
- Start: token.Start + p1,
- End: token.Start + p2,
- Term: token.Term[p1:p2],
- Position: token.Position,
- Type: analysis.AlphaNumeric,
- }
- ret = append(ret, t)
+func (codeTokenizer) Tokenize(input []byte) analysis.TokenStream {
+ lower := bytes.ToLower(input)
+ seen := make(map[string]bool)
+ var tokens analysis.TokenStream
+ var starts [3]int
+ for end, count := 0, 0; end < len(lower); {
+ starts[0], starts[1], starts[2] = starts[1], starts[2], end
+ count = min(count+1, len(starts))
+ _, size := utf8.DecodeRune(lower[end:])
+ end += size
+ for _, start := range starts[len(starts)-count:] {
+ if gram := lower[start:end]; !seen[string(gram)] {
+ seen[string(gram)] = true
+ tokens = append(tokens, &analysis.Token{Start: start, End: end, Term: gram, Position: len(tokens) + 1, Type: analysis.AlphaNumeric})
}
}
}
- return ret
+ return tokens
}
-func codeTokenFilterConstructor(_ map[string]any, _ *registry.Cache) (analysis.TokenFilter, error) {
- return &codeTokenFilter{
- re: regexp.MustCompile("[a-zA-Z]+|[0-9]+"),
- }, nil
+func codeTokenizerConstructor(_config map[string]any, _cache *registry.Cache) (analysis.Tokenizer, error) {
+ return codeTokenizer{}, nil
}
diff --git a/modules/indexer/code/elasticsearch/elasticsearch.go b/modules/indexer/code/elasticsearch/elasticsearch.go
index 74d749cd1d7..1e336ced05a 100644
--- a/modules/indexer/code/elasticsearch/elasticsearch.go
+++ b/modules/indexer/code/elasticsearch/elasticsearch.go
@@ -9,6 +9,7 @@ import (
"io"
"strconv"
"strings"
+ "unicode/utf8"
repo_model "gitea.dev/models/repo"
"gitea.dev/modules/analyze"
@@ -23,12 +24,11 @@ import (
"gitea.dev/modules/setting"
"gitea.dev/modules/timeutil"
"gitea.dev/modules/typesniffer"
- "gitea.dev/modules/util"
"github.com/go-enry/go-enry/v2"
)
-const esRepoIndexerLatestVersion = 3
+const esRepoIndexerLatestVersion = 4
var _ internal.Indexer = &Indexer{}
@@ -55,6 +55,10 @@ const (
"tokenizer": "content_tokenizer",
"filter" : ["lowercase"]
},
+ "content_short_analyzer": {
+ "tokenizer": "content_short_tokenizer",
+ "filter" : ["lowercase"]
+ },
"filename_path_analyzer": {
"tokenizer": "path_tokenizer"
},
@@ -64,8 +68,14 @@ const (
},
"tokenizer": {
"content_tokenizer": {
- "type": "simple_pattern_split",
- "pattern": "[^a-zA-Z0-9]"
+ "type": "ngram",
+ "min_gram": 3,
+ "max_gram": 3
+ },
+ "content_short_tokenizer": {
+ "type": "ngram",
+ "min_gram": 1,
+ "max_gram": 2
},
"path_tokenizer": {
"type": "path_hierarchy",
@@ -87,7 +97,6 @@ const (
},
"filename": {
"type": "text",
- "term_vector": "with_positions_offsets",
"index": true,
"fields": {
"path": {
@@ -102,9 +111,16 @@ const (
},
"content": {
"type": "text",
- "term_vector": "with_positions_offsets",
"index": true,
- "analyzer": "content_analyzer"
+ "analyzer": "content_analyzer",
+ "fields": {
+ "short": {
+ "type": "text",
+ "analyzer": "content_short_analyzer",
+ "index_options": "docs",
+ "norms": false
+ }
+ }
},
"commit_id": {
"type": "keyword",
@@ -236,23 +252,7 @@ func (b *Indexer) doDelete(ctx context.Context, repoID int64) error {
return b.DeleteByQuery(ctx, es.TermsQuery("repo_id", repoID))
}
-// contentMatchIndexPos find words positions for start and the following end on content. It will
-// return the beginning position of the first start and the ending position of the
-// first end following the start string.
-// If not found any of the positions, it will return -1, -1.
-func contentMatchIndexPos(content, start, end string) (int, int) {
- startIdx := strings.Index(content, start)
- if startIdx < 0 {
- return -1, -1
- }
- endIdx := strings.Index(content[startIdx+len(start):], end)
- if endIdx < 0 {
- return -1, -1
- }
- return startIdx, (startIdx + len(start) + endIdx + len(end)) - 9 // remove the length since we give Content the original data
-}
-
-func convertResult(searchResult *es.SearchResponse, kw string, pageSize int) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
+func convertResult(searchResult *es.SearchResponse, terms []string, pageSize int) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
hits := make([]*internal.SearchResult, 0, pageSize)
for _, hit := range searchResult.Hits {
repoID, fileName := internal.ParseIndexerID(hit.ID)
@@ -270,22 +270,9 @@ func convertResult(searchResult *es.SearchResponse, kw string, pageSize int) (in
return 0, nil, nil, fmt.Errorf("unexpected field types in search hit %q", hit.ID)
}
- // FIXME: There is no way to get the position the keyword on the content currently on the same request.
- // So we get it from content, this may made the query slower. See
- // https://discuss.elastic.co/t/fetching-position-of-keyword-in-matched-document/94291
- var startIndex, endIndex int
- if c, ok := hit.Highlight["filename"]; ok && len(c) > 0 {
+ startIndex, endIndex := internal.ContentMatchIndexPos(content, terms)
+ if startIndex < 0 {
startIndex, endIndex = internal.FilenameMatchIndexPos(content)
- } else if c, ok := hit.Highlight["content"]; ok && len(c) > 0 {
- // FIXME: Since the highlighting content will include and for the keywords,
- // now we should find the positions. But how to avoid html content which contains the
- // and tags? If elastic search has handled that?
- startIndex, endIndex = contentMatchIndexPos(c[0], "", "")
- if startIndex == -1 {
- panic(fmt.Sprintf("1===%s,,,%#v,,,%s", kw, hit.Highlight, c[0]))
- }
- } else {
- panic(fmt.Sprintf("2===%#v", hit.Highlight))
}
hits = append(hits, &internal.SearchResult{
@@ -328,11 +315,19 @@ func extractAggs(searchResult *es.SearchResponse) []*internal.SearchResultLangua
// Search searches for codes and language stats by given conditions.
func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
- searchMode := util.IfZero(opts.SearchMode, b.SupportedSearchModes()[0].ModeValue)
- contentQuery := es.Query(es.NewMultiMatchQuery(opts.Keyword, "content").Type(es.MultiMatchTypeBestFields).Operator("and"))
- if searchMode == indexer.SearchModeExact {
- contentQuery = es.MatchPhraseQuery("content", opts.Keyword)
+ terms := internal.SearchTerms(opts)
+ if len(terms) == 0 {
+ return 0, nil, nil, nil
}
+ contentQuery := es.NewBoolQuery()
+ for _, term := range terms {
+ if utf8.RuneCountInString(term) < 3 {
+ contentQuery.Must(es.NewMultiMatchQuery(term, "content.short").Operator("and"))
+ } else {
+ contentQuery.Must(es.MatchPhraseQuery("content", term))
+ }
+ }
+
kwQuery := es.NewBoolQuery().Should(
contentQuery,
es.NewMultiMatchQuery(opts.Keyword, "filename^10").Type(es.MultiMatchTypePhrasePrefix),
@@ -343,7 +338,6 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
}
start, pageSize := opts.GetSkipTake()
- kw := "" + opts.Keyword + ""
languageAggs := map[string]any{
"language": map[string]any{
"terms": map[string]any{
@@ -353,15 +347,6 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
},
},
}
- // number_of_fragments=0 returns the full highlighted content (no fragmentation).
- highlight := map[string]any{
- "fields": map[string]any{
- "content": map[string]any{},
- "filename": map[string]any{},
- },
- "number_of_fragments": 0,
- "type": "fvh",
- }
sort := []es.SortField{
{Field: "_score", Desc: true},
{Field: "updated_at", Desc: false},
@@ -375,12 +360,11 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
Size: pageSize,
TrackTotal: true,
Aggregations: languageAggs,
- Highlight: highlight,
})
if err != nil {
return 0, nil, nil, err
}
- return convertResult(resp, kw, pageSize)
+ return convertResult(resp, terms, pageSize)
}
countResp, err := b.Indexer.Search(ctx, es.SearchRequest{
@@ -400,12 +384,11 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
From: start,
Size: pageSize,
TrackTotal: true,
- Highlight: highlight,
})
if err != nil {
return 0, nil, nil, err
}
- total, hits, _, err := convertResult(resp, kw, pageSize)
+ total, hits, _, err := convertResult(resp, terms, pageSize)
return total, hits, extractAggs(countResp), err
}
diff --git a/modules/indexer/code/elasticsearch/elasticsearch_test.go b/modules/indexer/code/elasticsearch/elasticsearch_test.go
deleted file mode 100644
index e8f1f202ce0..00000000000
--- a/modules/indexer/code/elasticsearch/elasticsearch_test.go
+++ /dev/null
@@ -1,16 +0,0 @@
-// Copyright 2020 The Gitea Authors. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-package elasticsearch
-
-import (
- "testing"
-
- "github.com/stretchr/testify/assert"
-)
-
-func TestIndexPos(t *testing.T) {
- startIdx, endIdx := contentMatchIndexPos("test index start and end", "start", "end")
- assert.Equal(t, 11, startIdx)
- assert.Equal(t, 15, endIdx)
-}
diff --git a/modules/indexer/code/indexer_test.go b/modules/indexer/code/indexer_test.go
index 3a5cad97d76..1ad87e57773 100644
--- a/modules/indexer/code/indexer_test.go
+++ b/modules/indexer/code/indexer_test.go
@@ -5,18 +5,19 @@ package code
import (
"context"
- "os"
+ "fmt"
"slices"
"testing"
"time"
"gitea.dev/models/db"
+ repo_model "gitea.dev/models/repo"
"gitea.dev/models/unittest"
+ "gitea.dev/modules/git"
indexer_module "gitea.dev/modules/indexer"
"gitea.dev/modules/indexer/code/bleve"
"gitea.dev/modules/indexer/code/elasticsearch"
"gitea.dev/modules/indexer/code/internal"
- "gitea.dev/modules/setting"
"gitea.dev/modules/test"
"gitea.dev/modules/util"
@@ -37,8 +38,29 @@ func TestMain(m *testing.M) {
unittest.MainTest(m)
}
+const unicodeSearchContent = `Поставщик
+日本語のテキスト
+हिन्दी
+ภาษาไทย
+İSTANBUL
+literal
+abc bcd cde def
+`
+
func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
t.Run(name, func(t *testing.T) {
+ repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 62})
+ require.NoError(t, git.ForceFastImport(t.Context(), repo, []git.FastImportCommit{{
+ Ref: "refs/heads/master",
+ Files: []git.FastImportFile{
+ {Path: "avocado.md", Content: "# repo1\n\npineaple pie of cucumber juice"},
+ {Path: "cucumber.md", Content: "Salad is good for your health"},
+ {Path: "example-file.js", Content: "console.log(\"Hello, World!\")"},
+ {Path: "ham.md", Content: "This is also not cheese"},
+ {Path: "potato/ham.md", Content: "This is not cheese"},
+ {Path: "unicode.txt", Content: unicodeSearchContent},
+ },
+ }}))
assert.NoError(t, setupRepositoryIndexes(t.Context(), indexer))
// Wait for the index to catch up: ES/OpenSearch make writes visible
// only after a refresh (default interval: 1s). Bleve is synchronous
@@ -57,6 +79,7 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
Langs int
SearchMode indexer_module.SearchModeType
Results []codeSearchResult
+ Match string
}{
// Search for an exact match on the contents of a file
// This scenario yields a single result (the file README.md on the repo '1')
@@ -89,6 +112,10 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
Filename: "README.md",
Content: "# repo1\n\nDescription for repo1",
},
+ {
+ Filename: "avocado.md",
+ Content: "# repo1\n\npineaple pie of cucumber juice",
+ },
},
},
// Search for an exact match on the contents of a file within the repo '2'.
@@ -191,7 +218,7 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
},
},
// Search for matches on the contents of files within the repo '62'.
- // This scenario yields two results (both are based on contents, the first one is an exact match where as the second is a 'fuzzy' one)
+ // This scenario yields two results (both are based on contents, the first one is an exact match where as the second is a words match)
{
RepoIDs: []int64{62},
Keyword: "This is not cheese",
@@ -209,10 +236,9 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
},
// Search for matches on the contents of files regardless of case.
{
- RepoIDs: nil,
- Keyword: "dESCRIPTION",
- Langs: 1,
- SearchMode: indexer_module.SearchModeFuzzy,
+ RepoIDs: nil,
+ Keyword: "dESCRIPTION",
+ Langs: 1,
Results: []codeSearchResult{
{
Filename: "README.md",
@@ -257,6 +283,13 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
},
},
},
+ {RepoIDs: []int64{62}, Keyword: "ставщ", Langs: 1, SearchMode: indexer_module.SearchModeExact, Results: []codeSearchResult{{Filename: "unicode.txt", Content: unicodeSearchContent}}},
+ {RepoIDs: []int64{62}, Keyword: "テキスト", Langs: 1, Results: []codeSearchResult{{Filename: "unicode.txt", Content: unicodeSearchContent}}},
+ {RepoIDs: []int64{62}, Keyword: "हिन्दी", Langs: 1, SearchMode: indexer_module.SearchModeExact, Results: []codeSearchResult{{Filename: "unicode.txt", Content: unicodeSearchContent}}},
+ {RepoIDs: []int64{62}, Keyword: "iSTANBUL", Langs: 1, SearchMode: indexer_module.SearchModeExact, Match: "İSTANBUL", Results: []codeSearchResult{{Filename: "unicode.txt", Content: unicodeSearchContent}}},
+ {RepoIDs: []int64{62}, Keyword: "literal", Langs: 1, Match: "literal", Results: []codeSearchResult{{Filename: "unicode.txt", Content: unicodeSearchContent}}},
+ {RepoIDs: []int64{62}, Keyword: "日本 ภาษา", Langs: 1, Match: "日本", Results: []codeSearchResult{{Filename: "unicode.txt", Content: unicodeSearchContent}}},
+ {RepoIDs: []int64{62}, Keyword: "abcdef", SearchMode: indexer_module.SearchModeExact},
}
for _, kw := range keywords {
@@ -272,14 +305,15 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
})
require.NoError(t, err)
require.Len(t, langs, kw.Langs)
+ require.EqualValues(t, len(kw.Results), total)
+ require.Len(t, res, len(kw.Results))
hits := make([]codeSearchResult, 0, len(res))
- if total > 0 {
- assert.NotEmpty(t, kw.Results, "The given scenario does not provide any expected results")
- }
-
for _, hit := range res {
+ if kw.Match != "" {
+ require.Equal(t, kw.Match, hit.Content[hit.StartIndex:hit.EndIndex])
+ }
hits = append(hits, codeSearchResult{
Filename: hit.Filename,
Content: hit.Content,
@@ -307,7 +341,6 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
func TestBleveIndexAndSearch(t *testing.T) {
unittest.PrepareTestEnv(t)
- defer test.MockVariableValue(&setting.Indexer.TypeBleveMaxFuzzniess, 2)()
dir := t.TempDir()
idx := bleve.NewIndexer(dir)
@@ -322,13 +355,9 @@ func TestBleveIndexAndSearch(t *testing.T) {
func TestESIndexAndSearch(t *testing.T) {
unittest.PrepareTestEnv(t)
- u := os.Getenv("TEST_INDEXER_CODE_ES_URL")
- if u == "" {
- t.SkipNow()
- return
- }
+ u := test.ExternalServiceHTTP(t, "TEST_ELASTICSEARCH_URL", "http://elasticsearch:9200")
- indexer := elasticsearch.NewIndexer(u, "gitea_codes")
+ indexer := elasticsearch.NewIndexer(u, fmt.Sprintf("test_code_indexer_%d", time.Now().UnixNano()))
if _, err := indexer.Init(t.Context()); err != nil {
if indexer != nil {
indexer.Close()
diff --git a/modules/indexer/code/internal/util.go b/modules/indexer/code/internal/util.go
index 3608dc4f12c..b55d9eefbd5 100644
--- a/modules/indexer/code/internal/util.go
+++ b/modules/indexer/code/internal/util.go
@@ -5,7 +5,10 @@ package internal
import (
"strings"
+ "unicode"
+ "unicode/utf8"
+ "gitea.dev/modules/indexer"
"gitea.dev/modules/indexer/internal"
"gitea.dev/modules/log"
)
@@ -46,3 +49,43 @@ func FilenameMatchIndexPos(content string) (int, int) {
}
return 0, len(content)
}
+
+func SearchTerms(opts *SearchOptions) []string {
+ if opts.Keyword == "" {
+ return nil
+ }
+ if opts.SearchMode == "" || opts.SearchMode == indexer.SearchModeExact {
+ return []string{opts.Keyword}
+ } else {
+ return strings.Fields(opts.Keyword)
+ }
+}
+
+// ContentMatchIndexPos returns the byte range of the earliest term in content, or -1, -1 unless all terms occur
+func ContentMatchIndexPos(content string, terms []string) (int, int) {
+ lowerContent := strings.ToLower(content)
+ start, end := -1, -1
+ for _, term := range terms {
+ lowerTerm := strings.ToLower(term)
+ position := strings.Index(lowerContent, lowerTerm)
+ if position < 0 {
+ return -1, -1
+ }
+ if start < 0 || position < start {
+ start, end = position, position+len(lowerTerm)
+ }
+ }
+ lowerOffset, originalStart := 0, -1
+ for offset := 0; offset < len(content); {
+ character, size := utf8.DecodeRuneInString(content[offset:])
+ if lowerOffset == start {
+ originalStart = offset
+ }
+ lowerOffset += utf8.RuneLen(unicode.ToLower(character))
+ offset += size
+ if lowerOffset == end {
+ return originalStart, offset
+ }
+ }
+ return -1, -1
+}
diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/description b/tests/gitea-repositories-meta/org42/search-by-path.git/description
deleted file mode 100644
index ffc40a9c48a..00000000000
--- a/tests/gitea-repositories-meta/org42/search-by-path.git/description
+++ /dev/null
@@ -1,9 +0,0 @@
-This repository will be used to test code search. The snippet below shows its directory structure
-
-.
-├── avocado.md
-├── cucumber.md
-├── ham.md
-├── potato
-| └── ham.md
-└── example-file.js
\ No newline at end of file
diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/post-receive b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/post-receive
deleted file mode 100755
index 4b3d452abcc..00000000000
--- a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/post-receive
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-ORI_DIR=`pwd`
-SHELL_FOLDER=$(cd "$(dirname "$0")";pwd)
-cd "$ORI_DIR"
-for i in `ls "$SHELL_FOLDER/post-receive.d"`; do
- sh "$SHELL_FOLDER/post-receive.d/$i"
-done
\ No newline at end of file
diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/post-receive.d/gitea b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/post-receive.d/gitea
deleted file mode 100755
index 43a948da3a9..00000000000
--- a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/post-receive.d/gitea
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/usr/bin/env bash
-"$GITEA_ROOT/gitea" hook --config="$GITEA_ROOT/$GITEA_CONF" post-receive
diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/pre-receive b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/pre-receive
deleted file mode 100755
index 41270130536..00000000000
--- a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/pre-receive
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-ORI_DIR=`pwd`
-SHELL_FOLDER=$(cd "$(dirname "$0")";pwd)
-cd "$ORI_DIR"
-for i in `ls "$SHELL_FOLDER/pre-receive.d"`; do
- sh "$SHELL_FOLDER/pre-receive.d/$i"
-done
\ No newline at end of file
diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/pre-receive.d/gitea b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/pre-receive.d/gitea
deleted file mode 100755
index 49d09406364..00000000000
--- a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/pre-receive.d/gitea
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/usr/bin/env bash
-"$GITEA_ROOT/gitea" hook --config="$GITEA_ROOT/$GITEA_CONF" pre-receive
diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/proc-receive b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/proc-receive
deleted file mode 100755
index af2808b0370..00000000000
--- a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/proc-receive
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-ORI_DIR=`pwd`
-SHELL_FOLDER=$(cd "$(dirname "$0")";pwd)
-cd "$ORI_DIR"
-for i in `ls "$SHELL_FOLDER/proc-receive.d"`; do
- sh "$SHELL_FOLDER/proc-receive.d/$i"
-done
diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/proc-receive.d/gitea b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/proc-receive.d/gitea
deleted file mode 100755
index 97521c62115..00000000000
--- a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/proc-receive.d/gitea
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/usr/bin/env bash
-"$GITEA_ROOT/gitea" hook --config="$GITEA_ROOT/$GITEA_CONF" proc-receive
diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/update b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/update
deleted file mode 100755
index c186fe4a18b..00000000000
--- a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/update
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-ORI_DIR=`pwd`
-SHELL_FOLDER=$(cd "$(dirname "$0")";pwd)
-cd "$ORI_DIR"
-for i in `ls "$SHELL_FOLDER/update.d"`; do
- sh "$SHELL_FOLDER/update.d/$i" $1 $2 $3
-done
\ No newline at end of file
diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/update.d/gitea b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/update.d/gitea
deleted file mode 100755
index 38101c24266..00000000000
--- a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/update.d/gitea
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/usr/bin/env bash
-"$GITEA_ROOT/gitea" hook --config="$GITEA_ROOT/$GITEA_CONF" update $1 $2 $3
diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/info/refs b/tests/gitea-repositories-meta/org42/search-by-path.git/info/refs
deleted file mode 100644
index 4adf83dda31..00000000000
--- a/tests/gitea-repositories-meta/org42/search-by-path.git/info/refs
+++ /dev/null
@@ -1,13 +0,0 @@
-90c1019714259b24fb81711d4416ac0f18667dfa refs/heads/DefaultBranch
-985f0301dba5e7b34be866819cd15ad3d8f508ee refs/heads/branch2
-65f1bf27bc3bf70f64657658635e66094edbcb4d refs/heads/develop
-65f1bf27bc3bf70f64657658635e66094edbcb4d refs/heads/feature/1
-78fb907e3a3309eae4fe8fef030874cebbf1cd5e refs/heads/home-md-img-check
-9f894b61946fd2f7b8b9d8e370e4d62f915522f5 refs/heads/master
-62fb502a7172d4453f0322a2cc85bddffa57f07a refs/heads/pr-to-update
-4649299398e4d39a5c09eb4f534df6f1e1eb87cc refs/heads/sub-home-md-img-check
-3fa2f829675543ecfc16b2891aebe8bf0608a8f4 refs/notes/commits
-4a357436d925b5c974181ff12a994538ddc5a269 refs/pull/2/head
-5f22f7d0d95d614d25a5b68592adb345a4b5c7fd refs/pull/3/head
-62fb502a7172d4453f0322a2cc85bddffa57f07a refs/pull/5/head
-65f1bf27bc3bf70f64657658635e66094edbcb4d refs/tags/v1.1
diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/objects/.keep b/tests/gitea-repositories-meta/org42/search-by-path.git/objects/.keep
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/objects/info/packs b/tests/gitea-repositories-meta/org42/search-by-path.git/objects/info/packs
deleted file mode 100644
index 9774923d2ea..00000000000
--- a/tests/gitea-repositories-meta/org42/search-by-path.git/objects/info/packs
+++ /dev/null
@@ -1,2 +0,0 @@
-P pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.pack
-
diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.bitmap b/tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.bitmap
deleted file mode 100644
index 39c02c29877..00000000000
Binary files a/tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.bitmap and /dev/null differ
diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.idx b/tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.idx
deleted file mode 100644
index 38d0e6b7226..00000000000
Binary files a/tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.idx and /dev/null differ
diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.pack b/tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.pack
deleted file mode 100644
index 06c0a899f36..00000000000
Binary files a/tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.pack and /dev/null differ
diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.rev b/tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.rev
deleted file mode 100644
index b06ecca825a..00000000000
Binary files a/tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.rev and /dev/null differ
diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/packed-refs b/tests/gitea-repositories-meta/org42/search-by-path.git/packed-refs
deleted file mode 100644
index 2334e3da489..00000000000
--- a/tests/gitea-repositories-meta/org42/search-by-path.git/packed-refs
+++ /dev/null
@@ -1,14 +0,0 @@
-# pack-refs with: peeled fully-peeled sorted
-90c1019714259b24fb81711d4416ac0f18667dfa refs/heads/DefaultBranch
-985f0301dba5e7b34be866819cd15ad3d8f508ee refs/heads/branch2
-65f1bf27bc3bf70f64657658635e66094edbcb4d refs/heads/develop
-65f1bf27bc3bf70f64657658635e66094edbcb4d refs/heads/feature/1
-78fb907e3a3309eae4fe8fef030874cebbf1cd5e refs/heads/home-md-img-check
-9f894b61946fd2f7b8b9d8e370e4d62f915522f5 refs/heads/master
-62fb502a7172d4453f0322a2cc85bddffa57f07a refs/heads/pr-to-update
-4649299398e4d39a5c09eb4f534df6f1e1eb87cc refs/heads/sub-home-md-img-check
-3fa2f829675543ecfc16b2891aebe8bf0608a8f4 refs/notes/commits
-4a357436d925b5c974181ff12a994538ddc5a269 refs/pull/2/head
-5f22f7d0d95d614d25a5b68592adb345a4b5c7fd refs/pull/3/head
-62fb502a7172d4453f0322a2cc85bddffa57f07a refs/pull/5/head
-65f1bf27bc3bf70f64657658635e66094edbcb4d refs/tags/v1.1
diff --git a/tests/gitea-repositories-meta/user2/glob.git/objects/.keep b/tests/gitea-repositories-meta/user2/glob.git/objects/.keep
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/gitea-repositories-meta/user2/glob.git/objects/48/06cb9df135782b818c968c2fadbd2c150d23d6 b/tests/gitea-repositories-meta/user2/glob.git/objects/48/06cb9df135782b818c968c2fadbd2c150d23d6
deleted file mode 100644
index a393a432828..00000000000
Binary files a/tests/gitea-repositories-meta/user2/glob.git/objects/48/06cb9df135782b818c968c2fadbd2c150d23d6 and /dev/null differ
diff --git a/tests/gitea-repositories-meta/user2/glob.git/objects/59/fee614e09d1f1cd1e15e4b2a7e9c8873a81498 b/tests/gitea-repositories-meta/user2/glob.git/objects/59/fee614e09d1f1cd1e15e4b2a7e9c8873a81498
deleted file mode 100644
index a55c8cc6e69..00000000000
Binary files a/tests/gitea-repositories-meta/user2/glob.git/objects/59/fee614e09d1f1cd1e15e4b2a7e9c8873a81498 and /dev/null differ
diff --git a/tests/gitea-repositories-meta/user2/glob.git/objects/7c/8ac2f8d82a1eb5f6aaece6629ff11015f91eb4 b/tests/gitea-repositories-meta/user2/glob.git/objects/7c/8ac2f8d82a1eb5f6aaece6629ff11015f91eb4
deleted file mode 100644
index d5176e68c63..00000000000
Binary files a/tests/gitea-repositories-meta/user2/glob.git/objects/7c/8ac2f8d82a1eb5f6aaece6629ff11015f91eb4 and /dev/null differ
diff --git a/tests/gitea-repositories-meta/user2/glob.git/objects/8e/592e636d27ac144f92f7fe8c33631cbdea594d b/tests/gitea-repositories-meta/user2/glob.git/objects/8e/592e636d27ac144f92f7fe8c33631cbdea594d
deleted file mode 100644
index 8034110ebcf..00000000000
Binary files a/tests/gitea-repositories-meta/user2/glob.git/objects/8e/592e636d27ac144f92f7fe8c33631cbdea594d and /dev/null differ
diff --git a/tests/gitea-repositories-meta/user2/glob.git/objects/95/aff026f99a9ab76fbd01decb63dd3dbc03e498 b/tests/gitea-repositories-meta/user2/glob.git/objects/95/aff026f99a9ab76fbd01decb63dd3dbc03e498
deleted file mode 100644
index 0883f2b3eb6..00000000000
Binary files a/tests/gitea-repositories-meta/user2/glob.git/objects/95/aff026f99a9ab76fbd01decb63dd3dbc03e498 and /dev/null differ
diff --git a/tests/gitea-repositories-meta/user2/glob.git/objects/ae/d1ffed24cc3cf9b80490795e893cae4bddd684 b/tests/gitea-repositories-meta/user2/glob.git/objects/ae/d1ffed24cc3cf9b80490795e893cae4bddd684
deleted file mode 100644
index 03fa05de22c..00000000000
Binary files a/tests/gitea-repositories-meta/user2/glob.git/objects/ae/d1ffed24cc3cf9b80490795e893cae4bddd684 and /dev/null differ
diff --git a/tests/gitea-repositories-meta/user2/glob.git/objects/bf/d6a6583f9a9ac59bd726c1df26c64a89427ede b/tests/gitea-repositories-meta/user2/glob.git/objects/bf/d6a6583f9a9ac59bd726c1df26c64a89427ede
deleted file mode 100644
index 9475433e667..00000000000
Binary files a/tests/gitea-repositories-meta/user2/glob.git/objects/bf/d6a6583f9a9ac59bd726c1df26c64a89427ede and /dev/null differ
diff --git a/tests/gitea-repositories-meta/user2/glob.git/objects/c8/eb3b6c767ccb68411d0a1f6c769be69fb4d95a b/tests/gitea-repositories-meta/user2/glob.git/objects/c8/eb3b6c767ccb68411d0a1f6c769be69fb4d95a
deleted file mode 100644
index 2b6297f726a..00000000000
--- a/tests/gitea-repositories-meta/user2/glob.git/objects/c8/eb3b6c767ccb68411d0a1f6c769be69fb4d95a
+++ /dev/null
@@ -1 +0,0 @@
-x+)JMU03d040031QH+(a:㆖o