feat: search all fields (#622)
* ✨ feat(search): search all feature - add Description field to Gist struct and index it - extend SearchGistMetadata with Description and Content - update Bleve and Meilisearch to index and search Description - modify ParseSearchQueryStr to parse description: and content: keywords - update templates and i18n for new search options * Fix test * Set content by default Signed-off-by: Thomas Miceli <tho.miceli@gmail.com> * Config to define default searchable fields Signed-off-by: Thomas Miceli <tho.miceli@gmail.com> --------- Signed-off-by: Thomas Miceli <tho.miceli@gmail.com> Co-authored-by: Thomas Miceli <tho.miceli@gmail.com>
This commit is contained in:
Webysther Sperandio
committed by
GitHub
GitHub
parent
5ad01a3304
commit
279da52899
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
|
||||
@@ -14,6 +15,7 @@ import (
|
||||
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
|
||||
"github.com/blevesearch/bleve/v2/search/query"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/thomiceli/opengist/internal/config"
|
||||
)
|
||||
|
||||
type BleveIndexer struct {
|
||||
@@ -116,18 +118,60 @@ func (i *BleveIndexer) Remove(gistID uint) error {
|
||||
return (*atomicIndexer.Load()).(*BleveIndexer).index.Delete(strconv.Itoa(int(gistID)))
|
||||
}
|
||||
|
||||
func (i *BleveIndexer) Search(queryStr string, queryMetadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
|
||||
// Search returns a list of Gist IDs that match the given search metadata.
|
||||
// The method returns an error if any.
|
||||
//
|
||||
// The queryMetadata parameter is used to filter the search results.
|
||||
// For example, passing a non-empty Username will search for gists whose
|
||||
// username matches the given string.
|
||||
//
|
||||
// If the "All" field in queryMetadata is non-empty, the method will
|
||||
// search across all metadata fields with OR logic. Otherwise, the method
|
||||
// will add each metadata field with AND logic.
|
||||
//
|
||||
// The page parameter is used to paginate the search results.
|
||||
// The method returns the total number of search results in the second return
|
||||
// value.
|
||||
//
|
||||
// The third return value is a map of language counts for the search results.
|
||||
// The language counts are computed by asking Bleve to return the top 10
|
||||
// facets for the "Languages" field.
|
||||
func (i *BleveIndexer) Search(metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
|
||||
var err error
|
||||
var indexerQuery query.Query
|
||||
if queryStr != "" {
|
||||
// Use match query with fuzzy matching for more flexible content search
|
||||
contentQuery := bleve.NewMatchQuery(queryStr)
|
||||
contentQuery.SetField("Content")
|
||||
contentQuery.SetFuzziness(2)
|
||||
indexerQuery = contentQuery
|
||||
} else {
|
||||
contentQuery := bleve.NewMatchAllQuery()
|
||||
indexerQuery = contentQuery
|
||||
var indexerQuery query.Query = bleve.NewMatchAllQuery()
|
||||
|
||||
// Query factory
|
||||
factoryQuery := func(field, value string) query.Query {
|
||||
query := bleve.NewMatchPhraseQuery(value)
|
||||
query.SetField(field)
|
||||
return query
|
||||
}
|
||||
|
||||
// Exact search
|
||||
addQuery := func(field, value string) {
|
||||
if value != "" && value != "." {
|
||||
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, factoryQuery(field, value))
|
||||
}
|
||||
}
|
||||
|
||||
// Exact+fuzzy query factory: exact match is boosted so it ranks above fuzzy-only matches
|
||||
factoryFuzzyQuery := func(field, value string) query.Query {
|
||||
exact := bleve.NewMatchPhraseQuery(value)
|
||||
exact.SetField(field)
|
||||
exact.SetBoost(2.0)
|
||||
|
||||
fuzzy := bleve.NewMatchQuery(value)
|
||||
fuzzy.SetField(field)
|
||||
fuzzy.SetFuzziness(2)
|
||||
|
||||
return bleve.NewDisjunctionQuery(exact, fuzzy)
|
||||
}
|
||||
|
||||
// Exact+fuzzy search
|
||||
addFuzzy := func(field, value string) {
|
||||
if value != "" && value != "." {
|
||||
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, factoryFuzzyQuery(field, value))
|
||||
}
|
||||
}
|
||||
|
||||
// Visibility filtering: show public gists (Visibility=0) OR user's own gists
|
||||
@@ -143,48 +187,58 @@ func (i *BleveIndexer) Search(queryStr string, queryMetadata SearchGistMetadata,
|
||||
accessQuery := bleve.NewDisjunctionQuery(publicQuery, userIdQuery)
|
||||
indexerQuery = bleve.NewConjunctionQuery(accessQuery, indexerQuery)
|
||||
|
||||
buildFieldQuery := func(field, value string) query.Query {
|
||||
switch field {
|
||||
case "Title", "Description", "Filenames", "Content":
|
||||
return factoryFuzzyQuery(field, value)
|
||||
case "Extensions":
|
||||
return factoryQuery(field, "."+value)
|
||||
default: // Username, Languages, Topics
|
||||
return factoryQuery(field, value)
|
||||
}
|
||||
}
|
||||
|
||||
// Handle "All" field - search across all metadata fields with OR logic
|
||||
if queryMetadata.All != "" {
|
||||
allQueries := make([]query.Query, 0)
|
||||
|
||||
// Create match phrase queries for each field
|
||||
fields := []struct {
|
||||
field string
|
||||
value string
|
||||
}{
|
||||
{"Username", queryMetadata.All},
|
||||
{"Title", queryMetadata.All},
|
||||
{"Extensions", "." + queryMetadata.All},
|
||||
{"Filenames", queryMetadata.All},
|
||||
{"Languages", queryMetadata.All},
|
||||
{"Topics", queryMetadata.All},
|
||||
if metadata.All != "" {
|
||||
allQueries := make([]query.Query, 0, len(AllSearchFields))
|
||||
for _, field := range AllSearchFields {
|
||||
allQueries = append(allQueries, buildFieldQuery(field, metadata.All))
|
||||
}
|
||||
|
||||
for _, f := range fields {
|
||||
q := bleve.NewMatchPhraseQuery(f.value)
|
||||
q.FieldVal = f.field
|
||||
allQueries = append(allQueries, q)
|
||||
}
|
||||
|
||||
// Combine all field queries with OR (disjunction)
|
||||
allDisjunction := bleve.NewDisjunctionQuery(allQueries...)
|
||||
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, allDisjunction)
|
||||
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, bleve.NewDisjunctionQuery(allQueries...))
|
||||
} else {
|
||||
// Original behavior: add each metadata field with AND logic
|
||||
addQuery := func(field, value string) {
|
||||
if value != "" && value != "." {
|
||||
q := bleve.NewMatchPhraseQuery(value)
|
||||
q.FieldVal = field
|
||||
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, q)
|
||||
addQuery("Username", metadata.Username)
|
||||
addFuzzy("Title", metadata.Title)
|
||||
addFuzzy("Description", metadata.Description)
|
||||
addQuery("Extensions", "."+metadata.Extension)
|
||||
addFuzzy("Filenames", metadata.Filename)
|
||||
addQuery("Languages", metadata.Language)
|
||||
addQuery("Topics", metadata.Topic)
|
||||
addFuzzy("Content", metadata.Content)
|
||||
|
||||
// Handle default search fields from config with OR logic
|
||||
if metadata.Default != "" {
|
||||
var fields []string
|
||||
for _, f := range strings.Split(config.C.SearchDefault, ",") {
|
||||
f = strings.TrimSpace(f)
|
||||
if f == "all" {
|
||||
fields = AllSearchFields
|
||||
break
|
||||
}
|
||||
if indexField, ok := SearchFieldMap[f]; ok {
|
||||
fields = append(fields, indexField)
|
||||
}
|
||||
}
|
||||
if len(fields) == 1 {
|
||||
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, buildFieldQuery(fields[0], metadata.Default))
|
||||
} else if len(fields) > 1 {
|
||||
defaultQueries := make([]query.Query, 0, len(fields))
|
||||
for _, field := range fields {
|
||||
defaultQueries = append(defaultQueries, buildFieldQuery(field, metadata.Default))
|
||||
}
|
||||
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, bleve.NewDisjunctionQuery(defaultQueries...))
|
||||
}
|
||||
}
|
||||
|
||||
addQuery("Username", queryMetadata.Username)
|
||||
addQuery("Title", queryMetadata.Title)
|
||||
addQuery("Extensions", "."+queryMetadata.Extension)
|
||||
addQuery("Filenames", queryMetadata.Filename)
|
||||
addQuery("Languages", queryMetadata.Language)
|
||||
addQuery("Topics", queryMetadata.Topic)
|
||||
}
|
||||
|
||||
languageFacet := bleve.NewFacetRequest("Languages", 10)
|
||||
@@ -197,6 +251,8 @@ func (i *BleveIndexer) Search(queryStr string, queryMetadata SearchGistMetadata,
|
||||
s.Fields = []string{"GistID"}
|
||||
s.IncludeLocations = false
|
||||
|
||||
log.Debug().Interface("searchRequest", s).Msg("Bleve search request")
|
||||
|
||||
results, err := (*atomicIndexer.Load()).(*BleveIndexer).index.Search(s)
|
||||
if err != nil {
|
||||
return nil, 0, nil, err
|
||||
|
||||
Reference in New Issue
Block a user