feat: search all fields (#622)
* ✨ feat(search): search all feature - add Description field to Gist struct and index it - extend SearchGistMetadata with Description and Content - update Bleve and Meilisearch to index and search Description - modify ParseSearchQueryStr to parse description: and content: keywords - update templates and i18n for new search options * Fix test * Set content by default Signed-off-by: Thomas Miceli <tho.miceli@gmail.com> * Config to define default searchable fields Signed-off-by: Thomas Miceli <tho.miceli@gmail.com> --------- Signed-off-by: Thomas Miceli <tho.miceli@gmail.com> Co-authored-by: Thomas Miceli <tho.miceli@gmail.com>
This commit is contained in:
committed by
GitHub
parent
5ad01a3304
commit
279da52899
@@ -32,6 +32,10 @@ index.meili.host:
|
||||
# Set the API key for the Meiliseach server
|
||||
index.meili.api-key:
|
||||
|
||||
# Set the default search fields. Can contain multiple fields (e.g., `content,username`).
|
||||
# Fields: content,user,title,description,filename,extension,language,topic. Default: content
|
||||
search.default: content
|
||||
|
||||
# Default branch name used by Opengist when initializing Git repositories.
|
||||
# If not set, uses the Git default branch name. See https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup#_new_default_branch
|
||||
git.default-branch:
|
||||
|
||||
@@ -15,6 +15,7 @@ aside: false
|
||||
| index | OG_INDEX | `bleve` | Define the code indexer (either `bleve`, `meilisearch`, or empty for no index). |
|
||||
| index.meili.host | OG_MEILI_HOST | none | Set the host for the Meiliseach server. |
|
||||
| index.meili.api-key | OG_MEILI_API_KEY | none | Set the API key for the Meiliseach server. |
|
||||
| search.default | OG_SEARCH_DEFAULT | `content` | Set the default search fields. Can contain multiple fields (e.g., `content,username`). Fields: `content,user,title,description,filename,extension,language,topic`. |
|
||||
| git.default-branch | OG_GIT_DEFAULT_BRANCH | none | Default branch name used by Opengist when initializing Git repositories. If not set, uses the Git default branch name. More info [here](https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup#_new_default_branch) |
|
||||
| sqlite.journal-mode | OG_SQLITE_JOURNAL_MODE | `WAL` | Set the journal mode for SQLite. More info [here](https://www.sqlite.org/pragma.html#pragma_journal_mode) |
|
||||
| http.host | OG_HTTP_HOST | `0.0.0.0` | The host on which the HTTP server should bind. Use an IP address for network binding. Use a path for Unix socket binding (e.g. /run/opengist.sock) |
|
||||
|
||||
@@ -38,11 +38,12 @@ type config struct {
|
||||
DBUri string `yaml:"db-uri" env:"OG_DB_URI"`
|
||||
DBFilename string `yaml:"db-filename" env:"OG_DB_FILENAME"` // deprecated
|
||||
|
||||
IndexEnabled bool `yaml:"index.enabled" env:"OG_INDEX_ENABLED"` // deprecated
|
||||
Index string `yaml:"index" env:"OG_INDEX"`
|
||||
BleveDirname string `yaml:"index.dirname" env:"OG_INDEX_DIRNAME"` // deprecated
|
||||
MeiliHost string `yaml:"index.meili.host" env:"OG_MEILI_HOST"`
|
||||
MeiliAPIKey string `yaml:"index.meili.api-key" env:"OG_MEILI_API_KEY"`
|
||||
IndexEnabled bool `yaml:"index.enabled" env:"OG_INDEX_ENABLED"` // deprecated
|
||||
Index string `yaml:"index" env:"OG_INDEX"`
|
||||
BleveDirname string `yaml:"index.dirname" env:"OG_INDEX_DIRNAME"` // deprecated
|
||||
MeiliHost string `yaml:"index.meili.host" env:"OG_MEILI_HOST"`
|
||||
MeiliAPIKey string `yaml:"index.meili.api-key" env:"OG_MEILI_API_KEY"`
|
||||
SearchDefault string `yaml:"search.default" env:"OG_SEARCH_DEFAULT"`
|
||||
|
||||
GitDefaultBranch string `yaml:"git.default-branch" env:"OG_GIT_DEFAULT_BRANCH"`
|
||||
|
||||
@@ -111,6 +112,7 @@ func configWithDefaults() (*config, error) {
|
||||
c.OpengistHome = ""
|
||||
c.DBUri = "opengist.db"
|
||||
c.Index = "bleve"
|
||||
c.SearchDefault = "content"
|
||||
|
||||
c.SqliteJournalMode = "WAL"
|
||||
|
||||
|
||||
@@ -817,18 +817,19 @@ func (gist *Gist) ToIndexedGist() (*index.Gist, error) {
|
||||
}
|
||||
|
||||
indexedGist := &index.Gist{
|
||||
GistID: gist.ID,
|
||||
UserID: gist.UserID,
|
||||
Visibility: gist.Private.Uint(),
|
||||
Username: gist.User.Username,
|
||||
Title: gist.Title,
|
||||
Content: wholeContent,
|
||||
Filenames: fileNames,
|
||||
Extensions: exts,
|
||||
Languages: langs,
|
||||
Topics: topics,
|
||||
CreatedAt: gist.CreatedAt,
|
||||
UpdatedAt: gist.UpdatedAt,
|
||||
GistID: gist.ID,
|
||||
UserID: gist.UserID,
|
||||
Visibility: gist.Private.Uint(),
|
||||
Username: gist.User.Username,
|
||||
Description: gist.Description,
|
||||
Title: gist.Title,
|
||||
Content: wholeContent,
|
||||
Filenames: fileNames,
|
||||
Extensions: exts,
|
||||
Languages: langs,
|
||||
Topics: topics,
|
||||
CreatedAt: gist.CreatedAt,
|
||||
UpdatedAt: gist.UpdatedAt,
|
||||
}
|
||||
|
||||
return indexedGist, nil
|
||||
|
||||
@@ -88,10 +88,12 @@ gist.search.found: gists found
|
||||
gist.search.no-results: No gists found
|
||||
gist.search.help.user: gists created by user
|
||||
gist.search.help.title: gists with given title
|
||||
gist.search.help.description: gists with given description
|
||||
gist.search.help.filename: gists having files with given name
|
||||
gist.search.help.extension: gists having files with given extension
|
||||
gist.search.help.language: gists having files with given language
|
||||
gist.search.help.topic: gists with given topic
|
||||
gist.search.help.all: search all fields
|
||||
gist.search.placeholder.title: Title
|
||||
gist.search.placeholder.visibility: Visibility
|
||||
gist.search.placeholder.public: Public
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
|
||||
@@ -14,6 +15,7 @@ import (
|
||||
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
|
||||
"github.com/blevesearch/bleve/v2/search/query"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/thomiceli/opengist/internal/config"
|
||||
)
|
||||
|
||||
type BleveIndexer struct {
|
||||
@@ -116,18 +118,60 @@ func (i *BleveIndexer) Remove(gistID uint) error {
|
||||
return (*atomicIndexer.Load()).(*BleveIndexer).index.Delete(strconv.Itoa(int(gistID)))
|
||||
}
|
||||
|
||||
func (i *BleveIndexer) Search(queryStr string, queryMetadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
|
||||
// Search returns a list of Gist IDs that match the given search metadata.
|
||||
// The method returns an error if any.
|
||||
//
|
||||
// The queryMetadata parameter is used to filter the search results.
|
||||
// For example, passing a non-empty Username will search for gists whose
|
||||
// username matches the given string.
|
||||
//
|
||||
// If the "All" field in queryMetadata is non-empty, the method will
|
||||
// search across all metadata fields with OR logic. Otherwise, the method
|
||||
// will add each metadata field with AND logic.
|
||||
//
|
||||
// The page parameter is used to paginate the search results.
|
||||
// The method returns the total number of search results in the second return
|
||||
// value.
|
||||
//
|
||||
// The third return value is a map of language counts for the search results.
|
||||
// The language counts are computed by asking Bleve to return the top 10
|
||||
// facets for the "Languages" field.
|
||||
func (i *BleveIndexer) Search(metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
|
||||
var err error
|
||||
var indexerQuery query.Query
|
||||
if queryStr != "" {
|
||||
// Use match query with fuzzy matching for more flexible content search
|
||||
contentQuery := bleve.NewMatchQuery(queryStr)
|
||||
contentQuery.SetField("Content")
|
||||
contentQuery.SetFuzziness(2)
|
||||
indexerQuery = contentQuery
|
||||
} else {
|
||||
contentQuery := bleve.NewMatchAllQuery()
|
||||
indexerQuery = contentQuery
|
||||
var indexerQuery query.Query = bleve.NewMatchAllQuery()
|
||||
|
||||
// Query factory
|
||||
factoryQuery := func(field, value string) query.Query {
|
||||
query := bleve.NewMatchPhraseQuery(value)
|
||||
query.SetField(field)
|
||||
return query
|
||||
}
|
||||
|
||||
// Exact search
|
||||
addQuery := func(field, value string) {
|
||||
if value != "" && value != "." {
|
||||
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, factoryQuery(field, value))
|
||||
}
|
||||
}
|
||||
|
||||
// Exact+fuzzy query factory: exact match is boosted so it ranks above fuzzy-only matches
|
||||
factoryFuzzyQuery := func(field, value string) query.Query {
|
||||
exact := bleve.NewMatchPhraseQuery(value)
|
||||
exact.SetField(field)
|
||||
exact.SetBoost(2.0)
|
||||
|
||||
fuzzy := bleve.NewMatchQuery(value)
|
||||
fuzzy.SetField(field)
|
||||
fuzzy.SetFuzziness(2)
|
||||
|
||||
return bleve.NewDisjunctionQuery(exact, fuzzy)
|
||||
}
|
||||
|
||||
// Exact+fuzzy search
|
||||
addFuzzy := func(field, value string) {
|
||||
if value != "" && value != "." {
|
||||
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, factoryFuzzyQuery(field, value))
|
||||
}
|
||||
}
|
||||
|
||||
// Visibility filtering: show public gists (Visibility=0) OR user's own gists
|
||||
@@ -143,48 +187,58 @@ func (i *BleveIndexer) Search(queryStr string, queryMetadata SearchGistMetadata,
|
||||
accessQuery := bleve.NewDisjunctionQuery(publicQuery, userIdQuery)
|
||||
indexerQuery = bleve.NewConjunctionQuery(accessQuery, indexerQuery)
|
||||
|
||||
buildFieldQuery := func(field, value string) query.Query {
|
||||
switch field {
|
||||
case "Title", "Description", "Filenames", "Content":
|
||||
return factoryFuzzyQuery(field, value)
|
||||
case "Extensions":
|
||||
return factoryQuery(field, "."+value)
|
||||
default: // Username, Languages, Topics
|
||||
return factoryQuery(field, value)
|
||||
}
|
||||
}
|
||||
|
||||
// Handle "All" field - search across all metadata fields with OR logic
|
||||
if queryMetadata.All != "" {
|
||||
allQueries := make([]query.Query, 0)
|
||||
|
||||
// Create match phrase queries for each field
|
||||
fields := []struct {
|
||||
field string
|
||||
value string
|
||||
}{
|
||||
{"Username", queryMetadata.All},
|
||||
{"Title", queryMetadata.All},
|
||||
{"Extensions", "." + queryMetadata.All},
|
||||
{"Filenames", queryMetadata.All},
|
||||
{"Languages", queryMetadata.All},
|
||||
{"Topics", queryMetadata.All},
|
||||
if metadata.All != "" {
|
||||
allQueries := make([]query.Query, 0, len(AllSearchFields))
|
||||
for _, field := range AllSearchFields {
|
||||
allQueries = append(allQueries, buildFieldQuery(field, metadata.All))
|
||||
}
|
||||
|
||||
for _, f := range fields {
|
||||
q := bleve.NewMatchPhraseQuery(f.value)
|
||||
q.FieldVal = f.field
|
||||
allQueries = append(allQueries, q)
|
||||
}
|
||||
|
||||
// Combine all field queries with OR (disjunction)
|
||||
allDisjunction := bleve.NewDisjunctionQuery(allQueries...)
|
||||
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, allDisjunction)
|
||||
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, bleve.NewDisjunctionQuery(allQueries...))
|
||||
} else {
|
||||
// Original behavior: add each metadata field with AND logic
|
||||
addQuery := func(field, value string) {
|
||||
if value != "" && value != "." {
|
||||
q := bleve.NewMatchPhraseQuery(value)
|
||||
q.FieldVal = field
|
||||
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, q)
|
||||
addQuery("Username", metadata.Username)
|
||||
addFuzzy("Title", metadata.Title)
|
||||
addFuzzy("Description", metadata.Description)
|
||||
addQuery("Extensions", "."+metadata.Extension)
|
||||
addFuzzy("Filenames", metadata.Filename)
|
||||
addQuery("Languages", metadata.Language)
|
||||
addQuery("Topics", metadata.Topic)
|
||||
addFuzzy("Content", metadata.Content)
|
||||
|
||||
// Handle default search fields from config with OR logic
|
||||
if metadata.Default != "" {
|
||||
var fields []string
|
||||
for _, f := range strings.Split(config.C.SearchDefault, ",") {
|
||||
f = strings.TrimSpace(f)
|
||||
if f == "all" {
|
||||
fields = AllSearchFields
|
||||
break
|
||||
}
|
||||
if indexField, ok := SearchFieldMap[f]; ok {
|
||||
fields = append(fields, indexField)
|
||||
}
|
||||
}
|
||||
if len(fields) == 1 {
|
||||
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, buildFieldQuery(fields[0], metadata.Default))
|
||||
} else if len(fields) > 1 {
|
||||
defaultQueries := make([]query.Query, 0, len(fields))
|
||||
for _, field := range fields {
|
||||
defaultQueries = append(defaultQueries, buildFieldQuery(field, metadata.Default))
|
||||
}
|
||||
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, bleve.NewDisjunctionQuery(defaultQueries...))
|
||||
}
|
||||
}
|
||||
|
||||
addQuery("Username", queryMetadata.Username)
|
||||
addQuery("Title", queryMetadata.Title)
|
||||
addQuery("Extensions", "."+queryMetadata.Extension)
|
||||
addQuery("Filenames", queryMetadata.Filename)
|
||||
addQuery("Languages", queryMetadata.Language)
|
||||
addQuery("Topics", queryMetadata.Topic)
|
||||
}
|
||||
|
||||
languageFacet := bleve.NewFacetRequest("Languages", 10)
|
||||
@@ -197,6 +251,8 @@ func (i *BleveIndexer) Search(queryStr string, queryMetadata SearchGistMetadata,
|
||||
s.Fields = []string{"GistID"}
|
||||
s.IncludeLocations = false
|
||||
|
||||
log.Debug().Interface("searchRequest", s).Msg("Bleve search request")
|
||||
|
||||
results, err := (*atomicIndexer.Load()).(*BleveIndexer).index.Search(s)
|
||||
if err != nil {
|
||||
return nil, 0, nil, err
|
||||
|
||||
@@ -119,18 +119,19 @@ func TestBleveIndexerUnicodeSearch(t *testing.T) {
|
||||
|
||||
// Add a gist with Unicode content
|
||||
gist := &Gist{
|
||||
GistID: 100,
|
||||
UserID: 100,
|
||||
Visibility: 0,
|
||||
Username: "testuser",
|
||||
Title: "Unicode Test",
|
||||
Content: "Hello world with unicode characters: café résumé naïve",
|
||||
Filenames: []string{"test.txt"},
|
||||
Extensions: []string{".txt"},
|
||||
Languages: []string{"Text"},
|
||||
Topics: []string{"unicode"},
|
||||
CreatedAt: 1234567890,
|
||||
UpdatedAt: 1234567890,
|
||||
GistID: 100,
|
||||
UserID: 100,
|
||||
Visibility: 0,
|
||||
Username: "testuser",
|
||||
Title: "Unicode Test",
|
||||
Description: "Descrition with Unicode characters: Café résumé naive",
|
||||
Content: "Hello world with unicode characters: café résumé naïve",
|
||||
Filenames: []string{"test.txt"},
|
||||
Extensions: []string{".txt"},
|
||||
Languages: []string{"Text"},
|
||||
Topics: []string{"unicode"},
|
||||
CreatedAt: 1234567890,
|
||||
UpdatedAt: 1234567890,
|
||||
}
|
||||
|
||||
err := indexer.Add(gist)
|
||||
@@ -139,7 +140,7 @@ func TestBleveIndexerUnicodeSearch(t *testing.T) {
|
||||
}
|
||||
|
||||
// Search for unicode content
|
||||
gistIDs, total, _, err := indexer.Search("café", SearchGistMetadata{}, 100, 1)
|
||||
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "café"}, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Search failed: %v", err)
|
||||
}
|
||||
|
||||
@@ -1,26 +1,43 @@
|
||||
package index
|
||||
|
||||
var AllSearchFields = []string{"Username", "Title", "Description", "Filenames", "Extensions", "Languages", "Topics", "Content"}
|
||||
|
||||
var SearchFieldMap = map[string]string{
|
||||
"user": "Username",
|
||||
"title": "Title",
|
||||
"description": "Description",
|
||||
"filename": "Filenames",
|
||||
"extension": "Extensions",
|
||||
"language": "Languages",
|
||||
"topic": "Topics",
|
||||
"content": "Content",
|
||||
}
|
||||
|
||||
type Gist struct {
|
||||
GistID uint
|
||||
UserID uint
|
||||
Visibility uint
|
||||
Username string
|
||||
Title string
|
||||
Content string
|
||||
Filenames []string
|
||||
Extensions []string
|
||||
Languages []string
|
||||
Topics []string
|
||||
CreatedAt int64
|
||||
UpdatedAt int64
|
||||
GistID uint
|
||||
UserID uint
|
||||
Visibility uint
|
||||
Username string
|
||||
Description string
|
||||
Title string
|
||||
Content string
|
||||
Filenames []string
|
||||
Extensions []string
|
||||
Languages []string
|
||||
Topics []string
|
||||
CreatedAt int64
|
||||
UpdatedAt int64
|
||||
}
|
||||
|
||||
type SearchGistMetadata struct {
|
||||
Username string
|
||||
Title string
|
||||
Filename string
|
||||
Extension string
|
||||
Language string
|
||||
Topic string
|
||||
All string
|
||||
Username string
|
||||
Title string
|
||||
Description string
|
||||
Content string
|
||||
Filename string
|
||||
Extension string
|
||||
Language string
|
||||
Topic string
|
||||
All string
|
||||
Default string
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ type Indexer interface {
|
||||
Reset() error
|
||||
Add(gist *Gist) error
|
||||
Remove(gistID uint) error
|
||||
Search(query string, metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error)
|
||||
Search(metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error)
|
||||
}
|
||||
|
||||
type IndexerType string
|
||||
@@ -125,7 +125,11 @@ func RemoveFromIndex(gistID uint) error {
|
||||
return (*idx).Remove(gistID)
|
||||
}
|
||||
|
||||
func SearchGists(query string, metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
|
||||
// SearchGists returns a list of Gist IDs that match the given search metadata.
|
||||
// If the indexer is not enabled, it returns nil, 0, nil, nil.
|
||||
// If the indexer is not initialized, it returns nil, 0, nil, fmt.Errorf("indexer is not initialized").
|
||||
// The function returns an error if any.
|
||||
func SearchGists(metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
|
||||
if !IndexEnabled() {
|
||||
return nil, 0, nil, nil
|
||||
}
|
||||
@@ -135,7 +139,7 @@ func SearchGists(query string, metadata SearchGistMetadata, userId uint, page in
|
||||
return nil, 0, nil, fmt.Errorf("indexer is not initialized")
|
||||
}
|
||||
|
||||
return (*idx).Search(query, metadata, userId, page)
|
||||
return (*idx).Search(metadata, userId, page)
|
||||
}
|
||||
|
||||
func DepreactionIndexDirname() {
|
||||
|
||||
@@ -71,7 +71,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) {
|
||||
}
|
||||
|
||||
// Verify gist is searchable
|
||||
gistIDs, total, _, err := indexer.Search("test gist", SearchGistMetadata{}, 11, 1)
|
||||
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "test gist"}, 11, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Search failed: %v", err)
|
||||
}
|
||||
@@ -114,7 +114,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) {
|
||||
|
||||
// Search by Rust language
|
||||
metadata := SearchGistMetadata{Language: "Rust"}
|
||||
gistIDs, total, _, err := indexer.Search("", metadata, 11, 1)
|
||||
gistIDs, total, _, err := indexer.Search(metadata, 11, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Search by Rust language failed: %v", err)
|
||||
}
|
||||
@@ -156,7 +156,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) {
|
||||
}
|
||||
|
||||
// Search for unicode content
|
||||
_, total, _, err := indexer.Search("café", SearchGistMetadata{}, 11, 1)
|
||||
_, total, _, err := indexer.Search(SearchGistMetadata{All: "café"}, 11, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Search for unicode failed: %v", err)
|
||||
}
|
||||
@@ -189,7 +189,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) {
|
||||
}
|
||||
|
||||
// User 11 should see their own private gist
|
||||
gistIDs, total, _, err := indexer.Search("private gist", SearchGistMetadata{}, 11, 1)
|
||||
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "private gist"}, 11, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Search for private gist as owner failed: %v", err)
|
||||
}
|
||||
@@ -205,7 +205,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) {
|
||||
}
|
||||
|
||||
// User 999 should NOT see user 11's private gist
|
||||
gistIDs2, _, _, err := indexer.Search("private gist", SearchGistMetadata{}, 999, 1)
|
||||
gistIDs2, _, _, err := indexer.Search(SearchGistMetadata{All: "private gist"}, 999, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Search for private gist as other user failed: %v", err)
|
||||
}
|
||||
@@ -239,7 +239,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) {
|
||||
}
|
||||
|
||||
// Should still be searchable by content
|
||||
_, total, _, err := indexer.Search("Minimal", SearchGistMetadata{}, 11, 1)
|
||||
_, total, _, err := indexer.Search(SearchGistMetadata{All: "Minimal"}, 11, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Search for minimal gist failed: %v", err)
|
||||
}
|
||||
@@ -292,7 +292,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) {
|
||||
}
|
||||
|
||||
// Search should find updated content, not original
|
||||
gistIDs, total, _, err := indexer.Search("new information", SearchGistMetadata{}, 11, 1)
|
||||
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "new information"}, 11, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Search for updated content failed: %v", err)
|
||||
}
|
||||
@@ -308,7 +308,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) {
|
||||
}
|
||||
|
||||
// Old content should not be found
|
||||
gistIDsOld, _, _, _ := indexer.Search("Original", SearchGistMetadata{}, 11, 1)
|
||||
gistIDsOld, _, _, _ := indexer.Search(SearchGistMetadata{All: "Original"}, 11, 1)
|
||||
for _, id := range gistIDsOld {
|
||||
if id == 1006 {
|
||||
t.Error("Should not find gist by old content after update")
|
||||
@@ -340,7 +340,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) {
|
||||
|
||||
// Search by username
|
||||
metadata := SearchGistMetadata{Username: "newuser"}
|
||||
gistIDs, total, _, err := indexer.Search("", metadata, 12, 1)
|
||||
gistIDs, total, _, err := indexer.Search(metadata, 12, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Search by username failed: %v", err)
|
||||
}
|
||||
@@ -383,7 +383,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) {
|
||||
|
||||
// Search by one of the topics
|
||||
metadata := SearchGistMetadata{Topic: "microservices"}
|
||||
gistIDs, total, _, err := indexer.Search("", metadata, 11, 1)
|
||||
gistIDs, total, _, err := indexer.Search(metadata, 11, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Search by topic failed: %v", err)
|
||||
}
|
||||
@@ -427,7 +427,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) {
|
||||
}
|
||||
|
||||
// Search for content from the middle
|
||||
gistIDs, total, _, err := indexer.Search("Line 500", SearchGistMetadata{}, 11, 1)
|
||||
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "Line 500"}, 11, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Search in long content failed: %v", err)
|
||||
}
|
||||
@@ -451,7 +451,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
|
||||
|
||||
// Test 1: Search by content - all init gists have "searchable content"
|
||||
t.Run("SearchByContent", func(t *testing.T) {
|
||||
gistIDs, total, languageCounts, err := indexer.Search("searchable", SearchGistMetadata{}, 1, 1)
|
||||
gistIDs, total, languageCounts, err := indexer.Search(SearchGistMetadata{All: "searchable"}, 1, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Search failed: %v", err)
|
||||
}
|
||||
@@ -475,7 +475,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
|
||||
// Test 2: Search by specific language - Go
|
||||
t.Run("SearchByLanguage", func(t *testing.T) {
|
||||
metadata := SearchGistMetadata{Language: "Go"}
|
||||
_, total, _, err := indexer.Search("", metadata, 1, 1)
|
||||
_, total, _, err := indexer.Search(metadata, 1, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Search by language failed: %v", err)
|
||||
}
|
||||
@@ -489,7 +489,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
|
||||
// Test 3: Search by specific username - alice
|
||||
t.Run("SearchByUsername", func(t *testing.T) {
|
||||
metadata := SearchGistMetadata{Username: "alice"}
|
||||
_, total, _, err := indexer.Search("", metadata, 1, 1)
|
||||
_, total, _, err := indexer.Search(metadata, 1, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Search by username failed: %v", err)
|
||||
}
|
||||
@@ -503,7 +503,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
|
||||
// Test 4: Search by extension - Python (bob's private files)
|
||||
t.Run("SearchByExtension", func(t *testing.T) {
|
||||
metadata := SearchGistMetadata{Extension: "py"}
|
||||
_, total, _, err := indexer.Search("", metadata, 1, 1)
|
||||
_, total, _, err := indexer.Search(metadata, 1, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Search by extension failed: %v", err)
|
||||
}
|
||||
@@ -518,7 +518,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
|
||||
// Test 5: Search by topic - algorithms
|
||||
t.Run("SearchByTopic", func(t *testing.T) {
|
||||
metadata := SearchGistMetadata{Topic: "algorithms"}
|
||||
_, total, _, err := indexer.Search("", metadata, 1, 1)
|
||||
_, total, _, err := indexer.Search(metadata, 1, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Search by topic failed: %v", err)
|
||||
}
|
||||
@@ -535,7 +535,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
|
||||
Language: "Go",
|
||||
Username: "alice",
|
||||
}
|
||||
_, total, _, err := indexer.Search("", metadata, 1, 1)
|
||||
_, total, _, err := indexer.Search(metadata, 1, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Search with combined filters failed: %v", err)
|
||||
}
|
||||
@@ -548,7 +548,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
|
||||
|
||||
// Test 7: Search with no results
|
||||
t.Run("SearchNoResults", func(t *testing.T) {
|
||||
gistIDs, total, _, err := indexer.Search("nonexistentquery", SearchGistMetadata{}, 1, 1)
|
||||
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "nonexistentquery"}, 1, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Search with no results failed: %v", err)
|
||||
}
|
||||
@@ -562,7 +562,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
|
||||
|
||||
// Test 8: Empty query returns all accessible gists
|
||||
t.Run("SearchEmptyQuery", func(t *testing.T) {
|
||||
gistIDs, total, languageCounts, err := indexer.Search("", SearchGistMetadata{}, 1, 1)
|
||||
gistIDs, total, languageCounts, err := indexer.Search(SearchGistMetadata{}, 1, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Empty search failed: %v", err)
|
||||
}
|
||||
@@ -586,7 +586,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
|
||||
t.Run("SearchPagination", func(t *testing.T) {
|
||||
// As user 1, we have 334 gists total
|
||||
// Page 1
|
||||
gistIDs1, total, _, err := indexer.Search("searchable", SearchGistMetadata{}, 1, 1)
|
||||
gistIDs1, total, _, err := indexer.Search(SearchGistMetadata{All: "searchable"}, 1, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Page 1 search failed: %v", err)
|
||||
}
|
||||
@@ -598,7 +598,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
|
||||
}
|
||||
|
||||
// Page 2
|
||||
gistIDs2, _, _, err := indexer.Search("searchable", SearchGistMetadata{}, 1, 2)
|
||||
gistIDs2, _, _, err := indexer.Search(SearchGistMetadata{All: "searchable"}, 1, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("Page 2 search failed: %v", err)
|
||||
}
|
||||
@@ -619,7 +619,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
|
||||
// Search as user 2 (bob)
|
||||
// bob has 333 gists at i=1,4,7,... with visibility=1 (private)
|
||||
// As user 2, bob sees: alice's 334 public gists + bob's own 333 gists = 667 total
|
||||
_, total, _, err := indexer.Search("", SearchGistMetadata{}, 2, 1)
|
||||
_, total, _, err := indexer.Search(SearchGistMetadata{}, 2, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Search as user 2 failed: %v", err)
|
||||
}
|
||||
@@ -628,7 +628,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
|
||||
}
|
||||
|
||||
// Search as non-existent user (should only see public gists)
|
||||
_, totalPublic, _, err := indexer.Search("", SearchGistMetadata{}, 999, 1)
|
||||
_, totalPublic, _, err := indexer.Search(SearchGistMetadata{}, 999, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Search as user 999 failed: %v", err)
|
||||
}
|
||||
@@ -645,7 +645,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
|
||||
|
||||
// Test 11: Language facets validation
|
||||
t.Run("LanguageFacets", func(t *testing.T) {
|
||||
_, _, languageCounts, err := indexer.Search("", SearchGistMetadata{}, 1, 1)
|
||||
_, _, languageCounts, err := indexer.Search(SearchGistMetadata{}, 1, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Search for facets failed: %v", err)
|
||||
}
|
||||
@@ -755,7 +755,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) {
|
||||
// Test 1: All field matches username
|
||||
t.Run("AllFieldMatchesUsername", func(t *testing.T) {
|
||||
metadata := SearchGistMetadata{All: "testuser_unique"}
|
||||
gistIDs, total, _, err := indexer.Search("", metadata, 100, 1)
|
||||
gistIDs, total, _, err := indexer.Search(metadata, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("All field search failed: %v", err)
|
||||
}
|
||||
@@ -777,7 +777,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) {
|
||||
// Test 2: All field matches title
|
||||
t.Run("AllFieldMatchesTitle", func(t *testing.T) {
|
||||
metadata := SearchGistMetadata{All: "unique features"}
|
||||
gistIDs, total, _, err := indexer.Search("", metadata, 100, 1)
|
||||
gistIDs, total, _, err := indexer.Search(metadata, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("All field search failed: %v", err)
|
||||
}
|
||||
@@ -799,7 +799,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) {
|
||||
// Test 3: All field matches language
|
||||
t.Run("AllFieldMatchesLanguage", func(t *testing.T) {
|
||||
metadata := SearchGistMetadata{All: "Ruby"}
|
||||
gistIDs, total, _, err := indexer.Search("", metadata, 100, 1)
|
||||
gistIDs, total, _, err := indexer.Search(metadata, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("All field search failed: %v", err)
|
||||
}
|
||||
@@ -821,7 +821,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) {
|
||||
// Test 4: All field matches topic
|
||||
t.Run("AllFieldMatchesTopic", func(t *testing.T) {
|
||||
metadata := SearchGistMetadata{All: "unique_topic"}
|
||||
gistIDs, total, _, err := indexer.Search("", metadata, 100, 1)
|
||||
gistIDs, total, _, err := indexer.Search(metadata, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("All field search failed: %v", err)
|
||||
}
|
||||
@@ -843,7 +843,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) {
|
||||
// Test 5: All field matches extension
|
||||
t.Run("AllFieldMatchesExtension", func(t *testing.T) {
|
||||
metadata := SearchGistMetadata{All: "sh"}
|
||||
gistIDs, total, _, err := indexer.Search("", metadata, 100, 1)
|
||||
gistIDs, total, _, err := indexer.Search(metadata, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("All field search failed: %v", err)
|
||||
}
|
||||
@@ -865,7 +865,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) {
|
||||
// Test 6: All field matches filename
|
||||
t.Run("AllFieldMatchesFilename", func(t *testing.T) {
|
||||
metadata := SearchGistMetadata{All: "unique_file.rb"}
|
||||
gistIDs, total, _, err := indexer.Search("", metadata, 100, 1)
|
||||
gistIDs, total, _, err := indexer.Search(metadata, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("All field search failed: %v", err)
|
||||
}
|
||||
@@ -888,7 +888,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) {
|
||||
t.Run("AllFieldORBehavior", func(t *testing.T) {
|
||||
// "unique" appears in: username (3001), title (3002), topic (3003), filename (3004)
|
||||
metadata := SearchGistMetadata{All: "unique"}
|
||||
gistIDs, total, _, err := indexer.Search("", metadata, 100, 1)
|
||||
gistIDs, total, _, err := indexer.Search(metadata, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("All field OR search failed: %v", err)
|
||||
}
|
||||
@@ -916,14 +916,14 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) {
|
||||
t.Run("AllFieldVsSpecificField", func(t *testing.T) {
|
||||
// Search with All field
|
||||
metadataAll := SearchGistMetadata{All: "unique"}
|
||||
_, totalAll, _, err := indexer.Search("", metadataAll, 100, 1)
|
||||
_, totalAll, _, err := indexer.Search(metadataAll, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("All field search failed: %v", err)
|
||||
}
|
||||
|
||||
// Search with specific username field only
|
||||
metadataSpecific := SearchGistMetadata{Username: "testuser_unique"}
|
||||
_, totalSpecific, _, err := indexer.Search("", metadataSpecific, 100, 1)
|
||||
_, totalSpecific, _, err := indexer.Search(metadataSpecific, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Specific field search failed: %v", err)
|
||||
}
|
||||
@@ -936,8 +936,8 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) {
|
||||
|
||||
// Test 9: All field with no matches
|
||||
t.Run("AllFieldNoMatches", func(t *testing.T) {
|
||||
metadata := SearchGistMetadata{All: "nonexistentvalue12345"}
|
||||
gistIDs, total, _, err := indexer.Search("", metadata, 100, 1)
|
||||
metadata := SearchGistMetadata{All: "nonexistentvalue"}
|
||||
gistIDs, total, _, err := indexer.Search(metadata, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("All field no match search failed: %v", err)
|
||||
}
|
||||
@@ -957,7 +957,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) {
|
||||
Username: "nonexistent", // This should be ignored
|
||||
Language: "NonExistent", // This should be ignored
|
||||
}
|
||||
gistIDs, total, _, err := indexer.Search("", metadata, 100, 1)
|
||||
gistIDs, total, _, err := indexer.Search(metadata, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("All field with other fields search failed: %v", err)
|
||||
}
|
||||
@@ -983,7 +983,9 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) {
|
||||
// All field searches metadata, content query searches content
|
||||
// Both should work together
|
||||
metadata := SearchGistMetadata{All: "Ruby"}
|
||||
gistIDs, total, _, err := indexer.Search("examples", metadata, 100, 1)
|
||||
// Use All field for content search
|
||||
metadata.All = "examples"
|
||||
gistIDs, total, _, err := indexer.Search(metadata, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("All field with content query failed: %v", err)
|
||||
}
|
||||
@@ -1007,7 +1009,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) {
|
||||
t.Run("AllFieldCaseInsensitive", func(t *testing.T) {
|
||||
// Search with different case
|
||||
metadata := SearchGistMetadata{All: "RUBY"}
|
||||
gistIDs, total, _, err := indexer.Search("", metadata, 100, 1)
|
||||
gistIDs, total, _, err := indexer.Search(metadata, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("All field case insensitive search failed: %v", err)
|
||||
}
|
||||
@@ -1101,7 +1103,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) {
|
||||
|
||||
// Test 1: Exact match should work
|
||||
t.Run("ExactMatch", func(t *testing.T) {
|
||||
gistIDs, total, _, err := indexer.Search("algorithms", SearchGistMetadata{}, 100, 1)
|
||||
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "algorithms"}, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Exact match search failed: %v", err)
|
||||
}
|
||||
@@ -1123,7 +1125,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) {
|
||||
// Test 2: 1 character typo - substitution
|
||||
t.Run("OneCharSubstitution", func(t *testing.T) {
|
||||
// "algoritm" instead of "algorithm" (missing 'h')
|
||||
gistIDs, total, _, err := indexer.Search("algoritm", SearchGistMetadata{}, 100, 1)
|
||||
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "algoritm"}, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("1-char typo search failed: %v", err)
|
||||
}
|
||||
@@ -1145,7 +1147,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) {
|
||||
// Test 3: 1 character typo - deletion
|
||||
t.Run("OneCharDeletion", func(t *testing.T) {
|
||||
// "pythn" instead of "python" (missing 'o')
|
||||
gistIDs, total, _, err := indexer.Search("pythn", SearchGistMetadata{}, 100, 1)
|
||||
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "pythn"}, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("1-char deletion search failed: %v", err)
|
||||
}
|
||||
@@ -1167,7 +1169,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) {
|
||||
// Test 4: 1 character typo - insertion (extra character)
|
||||
t.Run("OneCharInsertion", func(t *testing.T) {
|
||||
// "pythonn" instead of "python" (extra 'n')
|
||||
gistIDs, total, _, err := indexer.Search("pythonn", SearchGistMetadata{}, 100, 1)
|
||||
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "pythonn"}, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("1-char insertion search failed: %v", err)
|
||||
}
|
||||
@@ -1189,7 +1191,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) {
|
||||
// Test 5: 2 character typos - should still match with fuzziness=2
|
||||
t.Run("TwoCharTypos", func(t *testing.T) {
|
||||
// "databse" instead of "database" (missing 'a', transposed 's')
|
||||
gistIDs, total, _, err := indexer.Search("databse", SearchGistMetadata{}, 100, 1)
|
||||
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "databse"}, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("2-char typo search failed: %v", err)
|
||||
}
|
||||
@@ -1211,7 +1213,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) {
|
||||
// Test 6: 2 character typos - different word
|
||||
t.Run("TwoCharTyposDifferentWord", func(t *testing.T) {
|
||||
// "javasript" instead of "javascript" (missing 'c')
|
||||
gistIDs, total, _, err := indexer.Search("javasript", SearchGistMetadata{}, 100, 1)
|
||||
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "javasript"}, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("2-char typo search failed: %v", err)
|
||||
}
|
||||
@@ -1233,7 +1235,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) {
|
||||
// Test 7: 3 character typos - should NOT match (beyond fuzziness=2)
|
||||
t.Run("ThreeCharTyposShouldNotMatch", func(t *testing.T) {
|
||||
// "algorthm" instead of "algorithm" (missing 'i', 't', 'h') - too different
|
||||
gistIDs, _, _, err := indexer.Search("algorthm", SearchGistMetadata{}, 100, 1)
|
||||
gistIDs, _, _, err := indexer.Search(SearchGistMetadata{All: "algorthm"}, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("3-char typo search failed: %v", err)
|
||||
}
|
||||
@@ -1256,7 +1258,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) {
|
||||
// Test 8: Transposition (swapped characters)
|
||||
t.Run("CharacterTransposition", func(t *testing.T) {
|
||||
// "pyhton" instead of "python" (swapped 'ht')
|
||||
gistIDs, total, _, err := indexer.Search("pyhton", SearchGistMetadata{}, 100, 1)
|
||||
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "pyhton"}, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Transposition search failed: %v", err)
|
||||
}
|
||||
@@ -1278,7 +1280,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) {
|
||||
// Test 9: Case insensitivity with fuzzy search
|
||||
t.Run("CaseInsensitiveWithFuzzy", func(t *testing.T) {
|
||||
// "PYTHN" (uppercase with typo)
|
||||
gistIDs, total, _, err := indexer.Search("PYTHN", SearchGistMetadata{}, 100, 1)
|
||||
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "PYTHN"}, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Case insensitive fuzzy search failed: %v", err)
|
||||
}
|
||||
@@ -1300,7 +1302,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) {
|
||||
// Test 10: Multiple words with typos
|
||||
t.Run("MultipleWordsWithTypos", func(t *testing.T) {
|
||||
// "relatonal databse" instead of "relational database"
|
||||
gistIDs, total, _, err := indexer.Search("relatonal databse", SearchGistMetadata{}, 100, 1)
|
||||
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "relatonal databse"}, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Multi-word fuzzy search failed: %v", err)
|
||||
}
|
||||
@@ -1322,7 +1324,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) {
|
||||
// Test 11: Short words with typos (edge case)
|
||||
t.Run("ShortWordsWithTypos", func(t *testing.T) {
|
||||
// "SLQ" instead of "SQL" (1 char typo on short word)
|
||||
gistIDs, total, _, err := indexer.Search("SLQ", SearchGistMetadata{}, 100, 1)
|
||||
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "SLQ"}, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Short word fuzzy search failed: %v", err)
|
||||
}
|
||||
@@ -1345,7 +1347,8 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) {
|
||||
t.Run("FuzzySearchWithMetadataFilters", func(t *testing.T) {
|
||||
// Search with typo AND username filter
|
||||
metadata := SearchGistMetadata{Username: "fuzzytest"}
|
||||
gistIDs, total, _, err := indexer.Search("algoritm", metadata, 100, 1)
|
||||
metadata.All = "algoritm"
|
||||
gistIDs, total, _, err := indexer.Search(metadata, 100, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Fuzzy search with metadata failed: %v", err)
|
||||
}
|
||||
@@ -1371,7 +1374,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
|
||||
// Test 1: Basic pagination - pages should be different
|
||||
t.Run("BasicPagination", func(t *testing.T) {
|
||||
// Search as user 1 (alice) - should see 334 public gists
|
||||
gistIDs1, total, _, err := indexer.Search("", SearchGistMetadata{}, 1, 1)
|
||||
gistIDs1, total, _, err := indexer.Search(SearchGistMetadata{}, 1, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Page 1 search failed: %v", err)
|
||||
}
|
||||
@@ -1382,7 +1385,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
|
||||
t.Fatal("Expected results on page 1")
|
||||
}
|
||||
|
||||
gistIDs2, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 2)
|
||||
gistIDs2, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("Page 2 search failed: %v", err)
|
||||
}
|
||||
@@ -1398,7 +1401,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
|
||||
|
||||
// Test 2: Page size - verify results per page (page size = 10)
|
||||
t.Run("PageSizeVerification", func(t *testing.T) {
|
||||
gistIDs1, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 1)
|
||||
gistIDs1, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Page 1 search failed: %v", err)
|
||||
}
|
||||
@@ -1410,11 +1413,11 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
|
||||
|
||||
// Test 3: Total count consistency across pages
|
||||
t.Run("TotalCountConsistency", func(t *testing.T) {
|
||||
_, total1, _, err := indexer.Search("", SearchGistMetadata{}, 1, 1)
|
||||
_, total1, _, err := indexer.Search(SearchGistMetadata{}, 1, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Page 1 search failed: %v", err)
|
||||
}
|
||||
_, total2, _, err := indexer.Search("", SearchGistMetadata{}, 1, 2)
|
||||
_, total2, _, err := indexer.Search(SearchGistMetadata{}, 1, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("Page 2 search failed: %v", err)
|
||||
}
|
||||
@@ -1429,7 +1432,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
|
||||
// Test 4: Out of bounds page
|
||||
t.Run("OutOfBoundsPage", func(t *testing.T) {
|
||||
// Page 100 is way beyond 334 results with page size 10
|
||||
gistIDs, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 100)
|
||||
gistIDs, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 100)
|
||||
if err != nil {
|
||||
t.Fatalf("Out of bounds page search failed: %v", err)
|
||||
}
|
||||
@@ -1440,7 +1443,8 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
|
||||
|
||||
// Test 5: Empty results pagination
|
||||
t.Run("EmptyResultsPagination", func(t *testing.T) {
|
||||
gistIDs, total, _, err := indexer.Search("nonexistentquery", SearchGistMetadata{}, 1, 1)
|
||||
metadata := SearchGistMetadata{All: "nonexistentquery"}
|
||||
gistIDs, total, _, err := indexer.Search(metadata, 1, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Empty search failed: %v", err)
|
||||
}
|
||||
@@ -1454,11 +1458,11 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
|
||||
|
||||
// Test 6: No duplicate IDs across pages (accounting for +1 overlap for hasMore indicator)
|
||||
t.Run("NoDuplicateIDs", func(t *testing.T) {
|
||||
gistIDs1, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 1)
|
||||
gistIDs1, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Page 1 search failed: %v", err)
|
||||
}
|
||||
gistIDs2, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 2)
|
||||
gistIDs2, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("Page 2 search failed: %v", err)
|
||||
}
|
||||
@@ -1489,7 +1493,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
|
||||
t.Run("PaginationWithFilters", func(t *testing.T) {
|
||||
// Filter by alice's username - should get 334 gists
|
||||
metadata := SearchGistMetadata{Username: "alice"}
|
||||
gistIDs1, total, _, err := indexer.Search("", metadata, 1, 1)
|
||||
gistIDs1, total, _, err := indexer.Search(metadata, 1, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Filtered page 1 search failed: %v", err)
|
||||
}
|
||||
@@ -1500,7 +1504,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
|
||||
t.Error("Expected results on filtered page 1")
|
||||
}
|
||||
|
||||
gistIDs2, _, _, err := indexer.Search("", metadata, 1, 2)
|
||||
gistIDs2, _, _, err := indexer.Search(metadata, 1, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("Filtered page 2 search failed: %v", err)
|
||||
}
|
||||
@@ -1518,7 +1522,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
|
||||
t.Run("LastPageVerification", func(t *testing.T) {
|
||||
// With 334 results and page size 10, page 34 should have 4 results
|
||||
// Let's just verify the last page has some results
|
||||
gistIDs34, total, _, err := indexer.Search("", SearchGistMetadata{}, 1, 34)
|
||||
gistIDs34, total, _, err := indexer.Search(SearchGistMetadata{}, 1, 34)
|
||||
if err != nil {
|
||||
t.Fatalf("Last page search failed: %v", err)
|
||||
}
|
||||
@@ -1530,7 +1534,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
|
||||
}
|
||||
|
||||
// Page 35 should be empty
|
||||
gistIDs35, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 35)
|
||||
gistIDs35, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 35)
|
||||
if err != nil {
|
||||
t.Fatalf("Beyond last page search failed: %v", err)
|
||||
}
|
||||
@@ -1541,15 +1545,15 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
|
||||
|
||||
// Test 9: Multiple pages have different results
|
||||
t.Run("MultiplePagesDifferent", func(t *testing.T) {
|
||||
gistIDs1, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 1)
|
||||
gistIDs1, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Page 1 search failed: %v", err)
|
||||
}
|
||||
gistIDs10, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 10)
|
||||
gistIDs10, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("Page 10 search failed: %v", err)
|
||||
}
|
||||
gistIDs20, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 20)
|
||||
gistIDs20, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 20)
|
||||
if err != nil {
|
||||
t.Fatalf("Page 20 search failed: %v", err)
|
||||
}
|
||||
@@ -1568,7 +1572,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
|
||||
// Test 10: Pagination with different users (visibility filtering)
|
||||
t.Run("PaginationWithVisibility", func(t *testing.T) {
|
||||
// User 2 (bob) sees 667 gists (334 public alice + 333 own private)
|
||||
gistIDs1Bob, totalBob, _, err := indexer.Search("", SearchGistMetadata{}, 2, 1)
|
||||
gistIDs1Bob, totalBob, _, err := indexer.Search(SearchGistMetadata{}, 2, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Bob page 1 search failed: %v", err)
|
||||
}
|
||||
@@ -1580,7 +1584,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
|
||||
}
|
||||
|
||||
// User 1 (alice) sees 334 gists
|
||||
_, totalAlice, _, err := indexer.Search("", SearchGistMetadata{}, 1, 1)
|
||||
_, totalAlice, _, err := indexer.Search(SearchGistMetadata{}, 1, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Alice page 1 search failed: %v", err)
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
"github.com/meilisearch/meilisearch-go"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/thomiceli/opengist/internal/config"
|
||||
)
|
||||
|
||||
type MeiliIndexer struct {
|
||||
@@ -63,9 +64,9 @@ func (i *MeiliIndexer) open() (meilisearch.IndexManager, error) {
|
||||
}
|
||||
|
||||
_, _ = i.client.Index(i.indexName).UpdateSettings(&meilisearch.Settings{
|
||||
FilterableAttributes: []string{"GistID", "UserID", "Visibility", "Username", "Title", "Filenames", "Extensions", "Languages", "Topics"},
|
||||
FilterableAttributes: []string{"GistID", "UserID", "Visibility", "Username", "Title", "Description", "Filenames", "Extensions", "Languages", "Topics"},
|
||||
DisplayedAttributes: []string{"GistID"},
|
||||
SearchableAttributes: []string{"Content", "Username", "Title", "Filenames", "Extensions", "Languages", "Topics"},
|
||||
SearchableAttributes: []string{"Content", "Username", "Title", "Description", "Filenames", "Extensions", "Languages", "Topics"},
|
||||
RankingRules: []string{"words"},
|
||||
})
|
||||
|
||||
@@ -109,7 +110,7 @@ func (i *MeiliIndexer) Remove(gistID uint) error {
|
||||
return err
|
||||
}
|
||||
|
||||
func (i *MeiliIndexer) Search(queryStr string, queryMetadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
|
||||
func (i *MeiliIndexer) Search(queryMetadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
|
||||
searchRequest := &meilisearch.SearchRequest{
|
||||
Offset: int64((page - 1) * 10),
|
||||
Limit: 11,
|
||||
@@ -128,6 +129,7 @@ func (i *MeiliIndexer) Search(queryStr string, queryMetadata SearchGistMetadata,
|
||||
}
|
||||
addFilter("Username", queryMetadata.Username)
|
||||
addFilter("Title", queryMetadata.Title)
|
||||
addFilter("Description", queryMetadata.Description)
|
||||
addFilter("Filenames", queryMetadata.Filename)
|
||||
addFilter("Extensions", queryMetadata.Extension)
|
||||
addFilter("Languages", queryMetadata.Language)
|
||||
@@ -137,7 +139,29 @@ func (i *MeiliIndexer) Search(queryStr string, queryMetadata SearchGistMetadata,
|
||||
searchRequest.Filter = strings.Join(filters, " AND ")
|
||||
}
|
||||
|
||||
response, err := (*atomicIndexer.Load()).(*MeiliIndexer).index.Search(queryStr, searchRequest)
|
||||
// build query string from provided metadata. Prefer `All`, then `Default`, fall back to `Content`.
|
||||
query := queryMetadata.All
|
||||
if query == "" && queryMetadata.Default != "" {
|
||||
query = queryMetadata.Default
|
||||
var fields []string
|
||||
for _, f := range strings.Split(config.C.SearchDefault, ",") {
|
||||
f = strings.TrimSpace(f)
|
||||
if f == "all" {
|
||||
fields = AllSearchFields
|
||||
break
|
||||
}
|
||||
if indexField, ok := SearchFieldMap[f]; ok {
|
||||
fields = append(fields, indexField)
|
||||
}
|
||||
}
|
||||
if len(fields) > 0 {
|
||||
searchRequest.AttributesToSearchOn = fields
|
||||
}
|
||||
} else if query == "" {
|
||||
query = queryMetadata.Content
|
||||
}
|
||||
|
||||
response, err := (*atomicIndexer.Load()).(*MeiliIndexer).index.Search(query, searchRequest)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("Failed to search Meilisearch index")
|
||||
return nil, 0, nil, err
|
||||
|
||||
@@ -164,6 +164,18 @@ func AllGists(ctx *context.Context) error {
|
||||
return ctx.Html("all.html")
|
||||
}
|
||||
|
||||
// Search handles the search page for gists.
|
||||
//
|
||||
// It takes a query parameter "q" which is a search query in the format:
|
||||
// "user:username title:title description:description filename:filename language:language topic:topic"
|
||||
//
|
||||
// It also takes a page parameter "page" which is the page number to display.
|
||||
//
|
||||
// It returns an error if the search query is invalid or if the page number is invalid.
|
||||
//
|
||||
// It returns the search results as a list of rendered gists, along with the total number of results, the languages found, and the search query.
|
||||
//
|
||||
// The search results are paginated, with 10 results per page.
|
||||
func Search(ctx *context.Context) error {
|
||||
var err error
|
||||
|
||||
@@ -171,7 +183,7 @@ func Search(ctx *context.Context) error {
|
||||
Query: ctx.QueryParam("q"),
|
||||
}
|
||||
|
||||
content, meta := handlers.ParseSearchQueryStr(ctx.QueryParam("q"))
|
||||
metadata := handlers.ParseSearchQueryStr(ctx.QueryParam("q"))
|
||||
pageInt := handlers.GetPage(ctx)
|
||||
|
||||
var currentUserId uint
|
||||
@@ -182,14 +194,18 @@ func Search(ctx *context.Context) error {
|
||||
currentUserId = 0
|
||||
}
|
||||
|
||||
gistsIds, nbHits, langs, err := index.SearchGists(content, index.SearchGistMetadata{
|
||||
Username: meta["user"],
|
||||
Title: meta["title"],
|
||||
Filename: meta["filename"],
|
||||
Extension: meta["extension"],
|
||||
Language: meta["language"],
|
||||
Topic: meta["topic"],
|
||||
All: meta["all"],
|
||||
// Search gists in the index and fetch the gists IDs from the database
|
||||
gistsIds, nbHits, langs, err := index.SearchGists(index.SearchGistMetadata{
|
||||
Username: metadata["user"],
|
||||
Title: metadata["title"],
|
||||
Description: metadata["description"],
|
||||
Filename: metadata["filename"],
|
||||
Extension: metadata["extension"],
|
||||
Language: metadata["language"],
|
||||
Topic: metadata["topic"],
|
||||
Content: metadata["content"],
|
||||
All: metadata["all"],
|
||||
Default: metadata["default"],
|
||||
}, currentUserId, pageInt)
|
||||
if err != nil {
|
||||
return ctx.ErrorRes(500, "Error searching gists", err)
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/gorilla/schema"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/thomiceli/opengist/internal/web/context"
|
||||
)
|
||||
|
||||
@@ -119,10 +119,16 @@ func Paginate[T any](ctx *context.Context, data []*T, pageInt int, perPage int,
|
||||
return nil
|
||||
}
|
||||
|
||||
func ParseSearchQueryStr(query string) (string, map[string]string) {
|
||||
// ParseSearchQueryStr parses a search query string and returns a map of metadata.
|
||||
// The query string is split into words and each word is checked if it contains a colon (:).
|
||||
// If a word contains a colon, it is split into a key-value pair and added to the metadata map.
|
||||
// If a word does not contain a colon, it is added to an "all" key in the metadata map.
|
||||
// The "all" key is used to search all fields in the index.
|
||||
// The function returns the metadata map.
|
||||
func ParseSearchQueryStr(query string) map[string]string {
|
||||
words := strings.Fields(query)
|
||||
metadata := make(map[string]string)
|
||||
var contentBuilder strings.Builder
|
||||
var allFieldsBuilder strings.Builder
|
||||
|
||||
for _, word := range words {
|
||||
if strings.Contains(word, ":") {
|
||||
@@ -133,10 +139,18 @@ func ParseSearchQueryStr(query string) (string, map[string]string) {
|
||||
metadata[key] = value
|
||||
}
|
||||
} else {
|
||||
contentBuilder.WriteString(word + " ")
|
||||
// Add to content search by default
|
||||
allFieldsBuilder.WriteString(word + " ")
|
||||
}
|
||||
}
|
||||
|
||||
content := strings.TrimSpace(contentBuilder.String())
|
||||
return content, metadata
|
||||
// Set the default search field
|
||||
allContent := strings.TrimSpace(allFieldsBuilder.String())
|
||||
if allContent != "" {
|
||||
metadata["default"] = allContent
|
||||
}
|
||||
|
||||
log.Debug().Msgf("Metadata: %v", metadata)
|
||||
|
||||
return metadata
|
||||
}
|
||||
|
||||
@@ -147,7 +147,10 @@ func (s *Server) setFuncMap() {
|
||||
return dict, nil
|
||||
},
|
||||
"addMetadataToSearchQuery": func(input, key, value string) string {
|
||||
content, metadata := handlers.ParseSearchQueryStr(input)
|
||||
metadata := handlers.ParseSearchQueryStr(input)
|
||||
// extract free-text content (stored under "all") and remove it from metadata
|
||||
content := metadata["all"]
|
||||
delete(metadata, "all")
|
||||
|
||||
metadata[key] = value
|
||||
|
||||
|
||||
2
templates/base/base_header.html
vendored
2
templates/base/base_header.html
vendored
@@ -116,10 +116,12 @@
|
||||
<div class="p-4 text-xs space-y-1">
|
||||
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">user:thomas</code> {{ .locale.Tr "gist.search.help.user" }}</p>
|
||||
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">title:mygist</code> {{ .locale.Tr "gist.search.help.title" }}</p>
|
||||
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">description:sync</code> {{ .locale.Tr "gist.search.help.description" }}</p>
|
||||
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">filename:myfile.txt</code> {{ .locale.Tr "gist.search.help.filename" }}</p>
|
||||
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">extension:yml</code> {{ .locale.Tr "gist.search.help.extension" }}</p>
|
||||
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">language:go</code> {{ .locale.Tr "gist.search.help.language" }}</p>
|
||||
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">topic:homelab</code> {{ .locale.Tr "gist.search.help.topic" }}</p>
|
||||
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">all:systemctl</code> {{ .locale.Tr "gist.search.help.all" }}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
Reference in New Issue
Block a user