diff --git a/config.yml b/config.yml index eaaa6ef..59b3820 100644 --- a/config.yml +++ b/config.yml @@ -32,6 +32,10 @@ index.meili.host: # Set the API key for the Meiliseach server index.meili.api-key: +# Set the default search fields. Can contain multiple fields (e.g., `content,username`). +# Fields: content,user,title,description,filename,extension,language,topic. Default: content +search.default: content + # Default branch name used by Opengist when initializing Git repositories. # If not set, uses the Git default branch name. See https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup#_new_default_branch git.default-branch: diff --git a/docs/configuration/cheat-sheet.md b/docs/configuration/cheat-sheet.md index f2d317e..ce6d5f1 100644 --- a/docs/configuration/cheat-sheet.md +++ b/docs/configuration/cheat-sheet.md @@ -15,6 +15,7 @@ aside: false | index | OG_INDEX | `bleve` | Define the code indexer (either `bleve`, `meilisearch`, or empty for no index). | | index.meili.host | OG_MEILI_HOST | none | Set the host for the Meiliseach server. | | index.meili.api-key | OG_MEILI_API_KEY | none | Set the API key for the Meiliseach server. | +| search.default | OG_SEARCH_DEFAULT | `content` | Set the default search fields. Can contain multiple fields (e.g., `content,username`). Fields: `content,user,title,description,filename,extension,language,topic`. | | git.default-branch | OG_GIT_DEFAULT_BRANCH | none | Default branch name used by Opengist when initializing Git repositories. If not set, uses the Git default branch name. More info [here](https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup#_new_default_branch) | | sqlite.journal-mode | OG_SQLITE_JOURNAL_MODE | `WAL` | Set the journal mode for SQLite. More info [here](https://www.sqlite.org/pragma.html#pragma_journal_mode) | | http.host | OG_HTTP_HOST | `0.0.0.0` | The host on which the HTTP server should bind. Use an IP address for network binding. Use a path for Unix socket binding (e.g. /run/opengist.sock) | diff --git a/internal/config/config.go b/internal/config/config.go index 6b3f83e..a975aff 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -38,11 +38,12 @@ type config struct { DBUri string `yaml:"db-uri" env:"OG_DB_URI"` DBFilename string `yaml:"db-filename" env:"OG_DB_FILENAME"` // deprecated - IndexEnabled bool `yaml:"index.enabled" env:"OG_INDEX_ENABLED"` // deprecated - Index string `yaml:"index" env:"OG_INDEX"` - BleveDirname string `yaml:"index.dirname" env:"OG_INDEX_DIRNAME"` // deprecated - MeiliHost string `yaml:"index.meili.host" env:"OG_MEILI_HOST"` - MeiliAPIKey string `yaml:"index.meili.api-key" env:"OG_MEILI_API_KEY"` + IndexEnabled bool `yaml:"index.enabled" env:"OG_INDEX_ENABLED"` // deprecated + Index string `yaml:"index" env:"OG_INDEX"` + BleveDirname string `yaml:"index.dirname" env:"OG_INDEX_DIRNAME"` // deprecated + MeiliHost string `yaml:"index.meili.host" env:"OG_MEILI_HOST"` + MeiliAPIKey string `yaml:"index.meili.api-key" env:"OG_MEILI_API_KEY"` + SearchDefault string `yaml:"search.default" env:"OG_SEARCH_DEFAULT"` GitDefaultBranch string `yaml:"git.default-branch" env:"OG_GIT_DEFAULT_BRANCH"` @@ -111,6 +112,7 @@ func configWithDefaults() (*config, error) { c.OpengistHome = "" c.DBUri = "opengist.db" c.Index = "bleve" + c.SearchDefault = "content" c.SqliteJournalMode = "WAL" diff --git a/internal/db/gist.go b/internal/db/gist.go index 7f9eadd..11dc428 100644 --- a/internal/db/gist.go +++ b/internal/db/gist.go @@ -817,18 +817,19 @@ func (gist *Gist) ToIndexedGist() (*index.Gist, error) { } indexedGist := &index.Gist{ - GistID: gist.ID, - UserID: gist.UserID, - Visibility: gist.Private.Uint(), - Username: gist.User.Username, - Title: gist.Title, - Content: wholeContent, - Filenames: fileNames, - Extensions: exts, - Languages: langs, - Topics: topics, - CreatedAt: gist.CreatedAt, - UpdatedAt: gist.UpdatedAt, + GistID: gist.ID, + UserID: gist.UserID, + Visibility: gist.Private.Uint(), + Username: gist.User.Username, + Description: gist.Description, + Title: gist.Title, + Content: wholeContent, + Filenames: fileNames, + Extensions: exts, + Languages: langs, + Topics: topics, + CreatedAt: gist.CreatedAt, + UpdatedAt: gist.UpdatedAt, } return indexedGist, nil diff --git a/internal/i18n/locales/en-US.yml b/internal/i18n/locales/en-US.yml index e686c5e..5e4d7ae 100644 --- a/internal/i18n/locales/en-US.yml +++ b/internal/i18n/locales/en-US.yml @@ -88,10 +88,12 @@ gist.search.found: gists found gist.search.no-results: No gists found gist.search.help.user: gists created by user gist.search.help.title: gists with given title +gist.search.help.description: gists with given description gist.search.help.filename: gists having files with given name gist.search.help.extension: gists having files with given extension gist.search.help.language: gists having files with given language gist.search.help.topic: gists with given topic +gist.search.help.all: search all fields gist.search.placeholder.title: Title gist.search.placeholder.visibility: Visibility gist.search.placeholder.public: Public diff --git a/internal/index/bleve.go b/internal/index/bleve.go index 4abef73..adb9171 100644 --- a/internal/index/bleve.go +++ b/internal/index/bleve.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "strconv" + "strings" "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" @@ -14,6 +15,7 @@ import ( "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" "github.com/blevesearch/bleve/v2/search/query" "github.com/rs/zerolog/log" + "github.com/thomiceli/opengist/internal/config" ) type BleveIndexer struct { @@ -116,18 +118,60 @@ func (i *BleveIndexer) Remove(gistID uint) error { return (*atomicIndexer.Load()).(*BleveIndexer).index.Delete(strconv.Itoa(int(gistID))) } -func (i *BleveIndexer) Search(queryStr string, queryMetadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) { +// Search returns a list of Gist IDs that match the given search metadata. +// The method returns an error if any. +// +// The queryMetadata parameter is used to filter the search results. +// For example, passing a non-empty Username will search for gists whose +// username matches the given string. +// +// If the "All" field in queryMetadata is non-empty, the method will +// search across all metadata fields with OR logic. Otherwise, the method +// will add each metadata field with AND logic. +// +// The page parameter is used to paginate the search results. +// The method returns the total number of search results in the second return +// value. +// +// The third return value is a map of language counts for the search results. +// The language counts are computed by asking Bleve to return the top 10 +// facets for the "Languages" field. +func (i *BleveIndexer) Search(metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) { var err error - var indexerQuery query.Query - if queryStr != "" { - // Use match query with fuzzy matching for more flexible content search - contentQuery := bleve.NewMatchQuery(queryStr) - contentQuery.SetField("Content") - contentQuery.SetFuzziness(2) - indexerQuery = contentQuery - } else { - contentQuery := bleve.NewMatchAllQuery() - indexerQuery = contentQuery + var indexerQuery query.Query = bleve.NewMatchAllQuery() + + // Query factory + factoryQuery := func(field, value string) query.Query { + query := bleve.NewMatchPhraseQuery(value) + query.SetField(field) + return query + } + + // Exact search + addQuery := func(field, value string) { + if value != "" && value != "." { + indexerQuery = bleve.NewConjunctionQuery(indexerQuery, factoryQuery(field, value)) + } + } + + // Exact+fuzzy query factory: exact match is boosted so it ranks above fuzzy-only matches + factoryFuzzyQuery := func(field, value string) query.Query { + exact := bleve.NewMatchPhraseQuery(value) + exact.SetField(field) + exact.SetBoost(2.0) + + fuzzy := bleve.NewMatchQuery(value) + fuzzy.SetField(field) + fuzzy.SetFuzziness(2) + + return bleve.NewDisjunctionQuery(exact, fuzzy) + } + + // Exact+fuzzy search + addFuzzy := func(field, value string) { + if value != "" && value != "." { + indexerQuery = bleve.NewConjunctionQuery(indexerQuery, factoryFuzzyQuery(field, value)) + } } // Visibility filtering: show public gists (Visibility=0) OR user's own gists @@ -143,48 +187,58 @@ func (i *BleveIndexer) Search(queryStr string, queryMetadata SearchGistMetadata, accessQuery := bleve.NewDisjunctionQuery(publicQuery, userIdQuery) indexerQuery = bleve.NewConjunctionQuery(accessQuery, indexerQuery) + buildFieldQuery := func(field, value string) query.Query { + switch field { + case "Title", "Description", "Filenames", "Content": + return factoryFuzzyQuery(field, value) + case "Extensions": + return factoryQuery(field, "."+value) + default: // Username, Languages, Topics + return factoryQuery(field, value) + } + } + // Handle "All" field - search across all metadata fields with OR logic - if queryMetadata.All != "" { - allQueries := make([]query.Query, 0) - - // Create match phrase queries for each field - fields := []struct { - field string - value string - }{ - {"Username", queryMetadata.All}, - {"Title", queryMetadata.All}, - {"Extensions", "." + queryMetadata.All}, - {"Filenames", queryMetadata.All}, - {"Languages", queryMetadata.All}, - {"Topics", queryMetadata.All}, + if metadata.All != "" { + allQueries := make([]query.Query, 0, len(AllSearchFields)) + for _, field := range AllSearchFields { + allQueries = append(allQueries, buildFieldQuery(field, metadata.All)) } - - for _, f := range fields { - q := bleve.NewMatchPhraseQuery(f.value) - q.FieldVal = f.field - allQueries = append(allQueries, q) - } - - // Combine all field queries with OR (disjunction) - allDisjunction := bleve.NewDisjunctionQuery(allQueries...) - indexerQuery = bleve.NewConjunctionQuery(indexerQuery, allDisjunction) + indexerQuery = bleve.NewConjunctionQuery(indexerQuery, bleve.NewDisjunctionQuery(allQueries...)) } else { // Original behavior: add each metadata field with AND logic - addQuery := func(field, value string) { - if value != "" && value != "." { - q := bleve.NewMatchPhraseQuery(value) - q.FieldVal = field - indexerQuery = bleve.NewConjunctionQuery(indexerQuery, q) + addQuery("Username", metadata.Username) + addFuzzy("Title", metadata.Title) + addFuzzy("Description", metadata.Description) + addQuery("Extensions", "."+metadata.Extension) + addFuzzy("Filenames", metadata.Filename) + addQuery("Languages", metadata.Language) + addQuery("Topics", metadata.Topic) + addFuzzy("Content", metadata.Content) + + // Handle default search fields from config with OR logic + if metadata.Default != "" { + var fields []string + for _, f := range strings.Split(config.C.SearchDefault, ",") { + f = strings.TrimSpace(f) + if f == "all" { + fields = AllSearchFields + break + } + if indexField, ok := SearchFieldMap[f]; ok { + fields = append(fields, indexField) + } + } + if len(fields) == 1 { + indexerQuery = bleve.NewConjunctionQuery(indexerQuery, buildFieldQuery(fields[0], metadata.Default)) + } else if len(fields) > 1 { + defaultQueries := make([]query.Query, 0, len(fields)) + for _, field := range fields { + defaultQueries = append(defaultQueries, buildFieldQuery(field, metadata.Default)) + } + indexerQuery = bleve.NewConjunctionQuery(indexerQuery, bleve.NewDisjunctionQuery(defaultQueries...)) } } - - addQuery("Username", queryMetadata.Username) - addQuery("Title", queryMetadata.Title) - addQuery("Extensions", "."+queryMetadata.Extension) - addQuery("Filenames", queryMetadata.Filename) - addQuery("Languages", queryMetadata.Language) - addQuery("Topics", queryMetadata.Topic) } languageFacet := bleve.NewFacetRequest("Languages", 10) @@ -197,6 +251,8 @@ func (i *BleveIndexer) Search(queryStr string, queryMetadata SearchGistMetadata, s.Fields = []string{"GistID"} s.IncludeLocations = false + log.Debug().Interface("searchRequest", s).Msg("Bleve search request") + results, err := (*atomicIndexer.Load()).(*BleveIndexer).index.Search(s) if err != nil { return nil, 0, nil, err diff --git a/internal/index/bleve_test.go b/internal/index/bleve_test.go index d2336d8..f07b5bb 100644 --- a/internal/index/bleve_test.go +++ b/internal/index/bleve_test.go @@ -119,18 +119,19 @@ func TestBleveIndexerUnicodeSearch(t *testing.T) { // Add a gist with Unicode content gist := &Gist{ - GistID: 100, - UserID: 100, - Visibility: 0, - Username: "testuser", - Title: "Unicode Test", - Content: "Hello world with unicode characters: café résumé naïve", - Filenames: []string{"test.txt"}, - Extensions: []string{".txt"}, - Languages: []string{"Text"}, - Topics: []string{"unicode"}, - CreatedAt: 1234567890, - UpdatedAt: 1234567890, + GistID: 100, + UserID: 100, + Visibility: 0, + Username: "testuser", + Title: "Unicode Test", + Description: "Descrition with Unicode characters: Café résumé naive", + Content: "Hello world with unicode characters: café résumé naïve", + Filenames: []string{"test.txt"}, + Extensions: []string{".txt"}, + Languages: []string{"Text"}, + Topics: []string{"unicode"}, + CreatedAt: 1234567890, + UpdatedAt: 1234567890, } err := indexer.Add(gist) @@ -139,7 +140,7 @@ func TestBleveIndexerUnicodeSearch(t *testing.T) { } // Search for unicode content - gistIDs, total, _, err := indexer.Search("café", SearchGistMetadata{}, 100, 1) + gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "café"}, 100, 1) if err != nil { t.Fatalf("Search failed: %v", err) } diff --git a/internal/index/gist.go b/internal/index/gist.go index 76943b8..6cb491c 100644 --- a/internal/index/gist.go +++ b/internal/index/gist.go @@ -1,26 +1,43 @@ package index +var AllSearchFields = []string{"Username", "Title", "Description", "Filenames", "Extensions", "Languages", "Topics", "Content"} + +var SearchFieldMap = map[string]string{ + "user": "Username", + "title": "Title", + "description": "Description", + "filename": "Filenames", + "extension": "Extensions", + "language": "Languages", + "topic": "Topics", + "content": "Content", +} + type Gist struct { - GistID uint - UserID uint - Visibility uint - Username string - Title string - Content string - Filenames []string - Extensions []string - Languages []string - Topics []string - CreatedAt int64 - UpdatedAt int64 + GistID uint + UserID uint + Visibility uint + Username string + Description string + Title string + Content string + Filenames []string + Extensions []string + Languages []string + Topics []string + CreatedAt int64 + UpdatedAt int64 } type SearchGistMetadata struct { - Username string - Title string - Filename string - Extension string - Language string - Topic string - All string + Username string + Title string + Description string + Content string + Filename string + Extension string + Language string + Topic string + All string + Default string } diff --git a/internal/index/indexer.go b/internal/index/indexer.go index 25907a8..e7f133f 100644 --- a/internal/index/indexer.go +++ b/internal/index/indexer.go @@ -17,7 +17,7 @@ type Indexer interface { Reset() error Add(gist *Gist) error Remove(gistID uint) error - Search(query string, metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) + Search(metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) } type IndexerType string @@ -125,7 +125,11 @@ func RemoveFromIndex(gistID uint) error { return (*idx).Remove(gistID) } -func SearchGists(query string, metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) { +// SearchGists returns a list of Gist IDs that match the given search metadata. +// If the indexer is not enabled, it returns nil, 0, nil, nil. +// If the indexer is not initialized, it returns nil, 0, nil, fmt.Errorf("indexer is not initialized"). +// The function returns an error if any. +func SearchGists(metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) { if !IndexEnabled() { return nil, 0, nil, nil } @@ -135,7 +139,7 @@ func SearchGists(query string, metadata SearchGistMetadata, userId uint, page in return nil, 0, nil, fmt.Errorf("indexer is not initialized") } - return (*idx).Search(query, metadata, userId, page) + return (*idx).Search(metadata, userId, page) } func DepreactionIndexDirname() { diff --git a/internal/index/indexer_test.go b/internal/index/indexer_test.go index a01a018..fe2a515 100644 --- a/internal/index/indexer_test.go +++ b/internal/index/indexer_test.go @@ -71,7 +71,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) { } // Verify gist is searchable - gistIDs, total, _, err := indexer.Search("test gist", SearchGistMetadata{}, 11, 1) + gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "test gist"}, 11, 1) if err != nil { t.Fatalf("Search failed: %v", err) } @@ -114,7 +114,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) { // Search by Rust language metadata := SearchGistMetadata{Language: "Rust"} - gistIDs, total, _, err := indexer.Search("", metadata, 11, 1) + gistIDs, total, _, err := indexer.Search(metadata, 11, 1) if err != nil { t.Fatalf("Search by Rust language failed: %v", err) } @@ -156,7 +156,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) { } // Search for unicode content - _, total, _, err := indexer.Search("café", SearchGistMetadata{}, 11, 1) + _, total, _, err := indexer.Search(SearchGistMetadata{All: "café"}, 11, 1) if err != nil { t.Fatalf("Search for unicode failed: %v", err) } @@ -189,7 +189,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) { } // User 11 should see their own private gist - gistIDs, total, _, err := indexer.Search("private gist", SearchGistMetadata{}, 11, 1) + gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "private gist"}, 11, 1) if err != nil { t.Fatalf("Search for private gist as owner failed: %v", err) } @@ -205,7 +205,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) { } // User 999 should NOT see user 11's private gist - gistIDs2, _, _, err := indexer.Search("private gist", SearchGistMetadata{}, 999, 1) + gistIDs2, _, _, err := indexer.Search(SearchGistMetadata{All: "private gist"}, 999, 1) if err != nil { t.Fatalf("Search for private gist as other user failed: %v", err) } @@ -239,7 +239,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) { } // Should still be searchable by content - _, total, _, err := indexer.Search("Minimal", SearchGistMetadata{}, 11, 1) + _, total, _, err := indexer.Search(SearchGistMetadata{All: "Minimal"}, 11, 1) if err != nil { t.Fatalf("Search for minimal gist failed: %v", err) } @@ -292,7 +292,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) { } // Search should find updated content, not original - gistIDs, total, _, err := indexer.Search("new information", SearchGistMetadata{}, 11, 1) + gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "new information"}, 11, 1) if err != nil { t.Fatalf("Search for updated content failed: %v", err) } @@ -308,7 +308,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) { } // Old content should not be found - gistIDsOld, _, _, _ := indexer.Search("Original", SearchGistMetadata{}, 11, 1) + gistIDsOld, _, _, _ := indexer.Search(SearchGistMetadata{All: "Original"}, 11, 1) for _, id := range gistIDsOld { if id == 1006 { t.Error("Should not find gist by old content after update") @@ -340,7 +340,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) { // Search by username metadata := SearchGistMetadata{Username: "newuser"} - gistIDs, total, _, err := indexer.Search("", metadata, 12, 1) + gistIDs, total, _, err := indexer.Search(metadata, 12, 1) if err != nil { t.Fatalf("Search by username failed: %v", err) } @@ -383,7 +383,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) { // Search by one of the topics metadata := SearchGistMetadata{Topic: "microservices"} - gistIDs, total, _, err := indexer.Search("", metadata, 11, 1) + gistIDs, total, _, err := indexer.Search(metadata, 11, 1) if err != nil { t.Fatalf("Search by topic failed: %v", err) } @@ -427,7 +427,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) { } // Search for content from the middle - gistIDs, total, _, err := indexer.Search("Line 500", SearchGistMetadata{}, 11, 1) + gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "Line 500"}, 11, 1) if err != nil { t.Fatalf("Search in long content failed: %v", err) } @@ -451,7 +451,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) { // Test 1: Search by content - all init gists have "searchable content" t.Run("SearchByContent", func(t *testing.T) { - gistIDs, total, languageCounts, err := indexer.Search("searchable", SearchGistMetadata{}, 1, 1) + gistIDs, total, languageCounts, err := indexer.Search(SearchGistMetadata{All: "searchable"}, 1, 1) if err != nil { t.Fatalf("Search failed: %v", err) } @@ -475,7 +475,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) { // Test 2: Search by specific language - Go t.Run("SearchByLanguage", func(t *testing.T) { metadata := SearchGistMetadata{Language: "Go"} - _, total, _, err := indexer.Search("", metadata, 1, 1) + _, total, _, err := indexer.Search(metadata, 1, 1) if err != nil { t.Fatalf("Search by language failed: %v", err) } @@ -489,7 +489,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) { // Test 3: Search by specific username - alice t.Run("SearchByUsername", func(t *testing.T) { metadata := SearchGistMetadata{Username: "alice"} - _, total, _, err := indexer.Search("", metadata, 1, 1) + _, total, _, err := indexer.Search(metadata, 1, 1) if err != nil { t.Fatalf("Search by username failed: %v", err) } @@ -503,7 +503,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) { // Test 4: Search by extension - Python (bob's private files) t.Run("SearchByExtension", func(t *testing.T) { metadata := SearchGistMetadata{Extension: "py"} - _, total, _, err := indexer.Search("", metadata, 1, 1) + _, total, _, err := indexer.Search(metadata, 1, 1) if err != nil { t.Fatalf("Search by extension failed: %v", err) } @@ -518,7 +518,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) { // Test 5: Search by topic - algorithms t.Run("SearchByTopic", func(t *testing.T) { metadata := SearchGistMetadata{Topic: "algorithms"} - _, total, _, err := indexer.Search("", metadata, 1, 1) + _, total, _, err := indexer.Search(metadata, 1, 1) if err != nil { t.Fatalf("Search by topic failed: %v", err) } @@ -535,7 +535,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) { Language: "Go", Username: "alice", } - _, total, _, err := indexer.Search("", metadata, 1, 1) + _, total, _, err := indexer.Search(metadata, 1, 1) if err != nil { t.Fatalf("Search with combined filters failed: %v", err) } @@ -548,7 +548,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) { // Test 7: Search with no results t.Run("SearchNoResults", func(t *testing.T) { - gistIDs, total, _, err := indexer.Search("nonexistentquery", SearchGistMetadata{}, 1, 1) + gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "nonexistentquery"}, 1, 1) if err != nil { t.Fatalf("Search with no results failed: %v", err) } @@ -562,7 +562,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) { // Test 8: Empty query returns all accessible gists t.Run("SearchEmptyQuery", func(t *testing.T) { - gistIDs, total, languageCounts, err := indexer.Search("", SearchGistMetadata{}, 1, 1) + gistIDs, total, languageCounts, err := indexer.Search(SearchGistMetadata{}, 1, 1) if err != nil { t.Fatalf("Empty search failed: %v", err) } @@ -586,7 +586,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) { t.Run("SearchPagination", func(t *testing.T) { // As user 1, we have 334 gists total // Page 1 - gistIDs1, total, _, err := indexer.Search("searchable", SearchGistMetadata{}, 1, 1) + gistIDs1, total, _, err := indexer.Search(SearchGistMetadata{All: "searchable"}, 1, 1) if err != nil { t.Fatalf("Page 1 search failed: %v", err) } @@ -598,7 +598,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) { } // Page 2 - gistIDs2, _, _, err := indexer.Search("searchable", SearchGistMetadata{}, 1, 2) + gistIDs2, _, _, err := indexer.Search(SearchGistMetadata{All: "searchable"}, 1, 2) if err != nil { t.Fatalf("Page 2 search failed: %v", err) } @@ -619,7 +619,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) { // Search as user 2 (bob) // bob has 333 gists at i=1,4,7,... with visibility=1 (private) // As user 2, bob sees: alice's 334 public gists + bob's own 333 gists = 667 total - _, total, _, err := indexer.Search("", SearchGistMetadata{}, 2, 1) + _, total, _, err := indexer.Search(SearchGistMetadata{}, 2, 1) if err != nil { t.Fatalf("Search as user 2 failed: %v", err) } @@ -628,7 +628,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) { } // Search as non-existent user (should only see public gists) - _, totalPublic, _, err := indexer.Search("", SearchGistMetadata{}, 999, 1) + _, totalPublic, _, err := indexer.Search(SearchGistMetadata{}, 999, 1) if err != nil { t.Fatalf("Search as user 999 failed: %v", err) } @@ -645,7 +645,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) { // Test 11: Language facets validation t.Run("LanguageFacets", func(t *testing.T) { - _, _, languageCounts, err := indexer.Search("", SearchGistMetadata{}, 1, 1) + _, _, languageCounts, err := indexer.Search(SearchGistMetadata{}, 1, 1) if err != nil { t.Fatalf("Search for facets failed: %v", err) } @@ -755,7 +755,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) { // Test 1: All field matches username t.Run("AllFieldMatchesUsername", func(t *testing.T) { metadata := SearchGistMetadata{All: "testuser_unique"} - gistIDs, total, _, err := indexer.Search("", metadata, 100, 1) + gistIDs, total, _, err := indexer.Search(metadata, 100, 1) if err != nil { t.Fatalf("All field search failed: %v", err) } @@ -777,7 +777,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) { // Test 2: All field matches title t.Run("AllFieldMatchesTitle", func(t *testing.T) { metadata := SearchGistMetadata{All: "unique features"} - gistIDs, total, _, err := indexer.Search("", metadata, 100, 1) + gistIDs, total, _, err := indexer.Search(metadata, 100, 1) if err != nil { t.Fatalf("All field search failed: %v", err) } @@ -799,7 +799,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) { // Test 3: All field matches language t.Run("AllFieldMatchesLanguage", func(t *testing.T) { metadata := SearchGistMetadata{All: "Ruby"} - gistIDs, total, _, err := indexer.Search("", metadata, 100, 1) + gistIDs, total, _, err := indexer.Search(metadata, 100, 1) if err != nil { t.Fatalf("All field search failed: %v", err) } @@ -821,7 +821,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) { // Test 4: All field matches topic t.Run("AllFieldMatchesTopic", func(t *testing.T) { metadata := SearchGistMetadata{All: "unique_topic"} - gistIDs, total, _, err := indexer.Search("", metadata, 100, 1) + gistIDs, total, _, err := indexer.Search(metadata, 100, 1) if err != nil { t.Fatalf("All field search failed: %v", err) } @@ -843,7 +843,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) { // Test 5: All field matches extension t.Run("AllFieldMatchesExtension", func(t *testing.T) { metadata := SearchGistMetadata{All: "sh"} - gistIDs, total, _, err := indexer.Search("", metadata, 100, 1) + gistIDs, total, _, err := indexer.Search(metadata, 100, 1) if err != nil { t.Fatalf("All field search failed: %v", err) } @@ -865,7 +865,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) { // Test 6: All field matches filename t.Run("AllFieldMatchesFilename", func(t *testing.T) { metadata := SearchGistMetadata{All: "unique_file.rb"} - gistIDs, total, _, err := indexer.Search("", metadata, 100, 1) + gistIDs, total, _, err := indexer.Search(metadata, 100, 1) if err != nil { t.Fatalf("All field search failed: %v", err) } @@ -888,7 +888,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) { t.Run("AllFieldORBehavior", func(t *testing.T) { // "unique" appears in: username (3001), title (3002), topic (3003), filename (3004) metadata := SearchGistMetadata{All: "unique"} - gistIDs, total, _, err := indexer.Search("", metadata, 100, 1) + gistIDs, total, _, err := indexer.Search(metadata, 100, 1) if err != nil { t.Fatalf("All field OR search failed: %v", err) } @@ -916,14 +916,14 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) { t.Run("AllFieldVsSpecificField", func(t *testing.T) { // Search with All field metadataAll := SearchGistMetadata{All: "unique"} - _, totalAll, _, err := indexer.Search("", metadataAll, 100, 1) + _, totalAll, _, err := indexer.Search(metadataAll, 100, 1) if err != nil { t.Fatalf("All field search failed: %v", err) } // Search with specific username field only metadataSpecific := SearchGistMetadata{Username: "testuser_unique"} - _, totalSpecific, _, err := indexer.Search("", metadataSpecific, 100, 1) + _, totalSpecific, _, err := indexer.Search(metadataSpecific, 100, 1) if err != nil { t.Fatalf("Specific field search failed: %v", err) } @@ -936,8 +936,8 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) { // Test 9: All field with no matches t.Run("AllFieldNoMatches", func(t *testing.T) { - metadata := SearchGistMetadata{All: "nonexistentvalue12345"} - gistIDs, total, _, err := indexer.Search("", metadata, 100, 1) + metadata := SearchGistMetadata{All: "nonexistentvalue"} + gistIDs, total, _, err := indexer.Search(metadata, 100, 1) if err != nil { t.Fatalf("All field no match search failed: %v", err) } @@ -957,7 +957,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) { Username: "nonexistent", // This should be ignored Language: "NonExistent", // This should be ignored } - gistIDs, total, _, err := indexer.Search("", metadata, 100, 1) + gistIDs, total, _, err := indexer.Search(metadata, 100, 1) if err != nil { t.Fatalf("All field with other fields search failed: %v", err) } @@ -983,7 +983,9 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) { // All field searches metadata, content query searches content // Both should work together metadata := SearchGistMetadata{All: "Ruby"} - gistIDs, total, _, err := indexer.Search("examples", metadata, 100, 1) + // Use All field for content search + metadata.All = "examples" + gistIDs, total, _, err := indexer.Search(metadata, 100, 1) if err != nil { t.Fatalf("All field with content query failed: %v", err) } @@ -1007,7 +1009,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) { t.Run("AllFieldCaseInsensitive", func(t *testing.T) { // Search with different case metadata := SearchGistMetadata{All: "RUBY"} - gistIDs, total, _, err := indexer.Search("", metadata, 100, 1) + gistIDs, total, _, err := indexer.Search(metadata, 100, 1) if err != nil { t.Fatalf("All field case insensitive search failed: %v", err) } @@ -1101,7 +1103,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) { // Test 1: Exact match should work t.Run("ExactMatch", func(t *testing.T) { - gistIDs, total, _, err := indexer.Search("algorithms", SearchGistMetadata{}, 100, 1) + gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "algorithms"}, 100, 1) if err != nil { t.Fatalf("Exact match search failed: %v", err) } @@ -1123,7 +1125,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) { // Test 2: 1 character typo - substitution t.Run("OneCharSubstitution", func(t *testing.T) { // "algoritm" instead of "algorithm" (missing 'h') - gistIDs, total, _, err := indexer.Search("algoritm", SearchGistMetadata{}, 100, 1) + gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "algoritm"}, 100, 1) if err != nil { t.Fatalf("1-char typo search failed: %v", err) } @@ -1145,7 +1147,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) { // Test 3: 1 character typo - deletion t.Run("OneCharDeletion", func(t *testing.T) { // "pythn" instead of "python" (missing 'o') - gistIDs, total, _, err := indexer.Search("pythn", SearchGistMetadata{}, 100, 1) + gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "pythn"}, 100, 1) if err != nil { t.Fatalf("1-char deletion search failed: %v", err) } @@ -1167,7 +1169,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) { // Test 4: 1 character typo - insertion (extra character) t.Run("OneCharInsertion", func(t *testing.T) { // "pythonn" instead of "python" (extra 'n') - gistIDs, total, _, err := indexer.Search("pythonn", SearchGistMetadata{}, 100, 1) + gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "pythonn"}, 100, 1) if err != nil { t.Fatalf("1-char insertion search failed: %v", err) } @@ -1189,7 +1191,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) { // Test 5: 2 character typos - should still match with fuzziness=2 t.Run("TwoCharTypos", func(t *testing.T) { // "databse" instead of "database" (missing 'a', transposed 's') - gistIDs, total, _, err := indexer.Search("databse", SearchGistMetadata{}, 100, 1) + gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "databse"}, 100, 1) if err != nil { t.Fatalf("2-char typo search failed: %v", err) } @@ -1211,7 +1213,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) { // Test 6: 2 character typos - different word t.Run("TwoCharTyposDifferentWord", func(t *testing.T) { // "javasript" instead of "javascript" (missing 'c') - gistIDs, total, _, err := indexer.Search("javasript", SearchGistMetadata{}, 100, 1) + gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "javasript"}, 100, 1) if err != nil { t.Fatalf("2-char typo search failed: %v", err) } @@ -1233,7 +1235,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) { // Test 7: 3 character typos - should NOT match (beyond fuzziness=2) t.Run("ThreeCharTyposShouldNotMatch", func(t *testing.T) { // "algorthm" instead of "algorithm" (missing 'i', 't', 'h') - too different - gistIDs, _, _, err := indexer.Search("algorthm", SearchGistMetadata{}, 100, 1) + gistIDs, _, _, err := indexer.Search(SearchGistMetadata{All: "algorthm"}, 100, 1) if err != nil { t.Fatalf("3-char typo search failed: %v", err) } @@ -1256,7 +1258,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) { // Test 8: Transposition (swapped characters) t.Run("CharacterTransposition", func(t *testing.T) { // "pyhton" instead of "python" (swapped 'ht') - gistIDs, total, _, err := indexer.Search("pyhton", SearchGistMetadata{}, 100, 1) + gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "pyhton"}, 100, 1) if err != nil { t.Fatalf("Transposition search failed: %v", err) } @@ -1278,7 +1280,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) { // Test 9: Case insensitivity with fuzzy search t.Run("CaseInsensitiveWithFuzzy", func(t *testing.T) { // "PYTHN" (uppercase with typo) - gistIDs, total, _, err := indexer.Search("PYTHN", SearchGistMetadata{}, 100, 1) + gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "PYTHN"}, 100, 1) if err != nil { t.Fatalf("Case insensitive fuzzy search failed: %v", err) } @@ -1300,7 +1302,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) { // Test 10: Multiple words with typos t.Run("MultipleWordsWithTypos", func(t *testing.T) { // "relatonal databse" instead of "relational database" - gistIDs, total, _, err := indexer.Search("relatonal databse", SearchGistMetadata{}, 100, 1) + gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "relatonal databse"}, 100, 1) if err != nil { t.Fatalf("Multi-word fuzzy search failed: %v", err) } @@ -1322,7 +1324,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) { // Test 11: Short words with typos (edge case) t.Run("ShortWordsWithTypos", func(t *testing.T) { // "SLQ" instead of "SQL" (1 char typo on short word) - gistIDs, total, _, err := indexer.Search("SLQ", SearchGistMetadata{}, 100, 1) + gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "SLQ"}, 100, 1) if err != nil { t.Fatalf("Short word fuzzy search failed: %v", err) } @@ -1345,7 +1347,8 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) { t.Run("FuzzySearchWithMetadataFilters", func(t *testing.T) { // Search with typo AND username filter metadata := SearchGistMetadata{Username: "fuzzytest"} - gistIDs, total, _, err := indexer.Search("algoritm", metadata, 100, 1) + metadata.All = "algoritm" + gistIDs, total, _, err := indexer.Search(metadata, 100, 1) if err != nil { t.Fatalf("Fuzzy search with metadata failed: %v", err) } @@ -1371,7 +1374,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) { // Test 1: Basic pagination - pages should be different t.Run("BasicPagination", func(t *testing.T) { // Search as user 1 (alice) - should see 334 public gists - gistIDs1, total, _, err := indexer.Search("", SearchGistMetadata{}, 1, 1) + gistIDs1, total, _, err := indexer.Search(SearchGistMetadata{}, 1, 1) if err != nil { t.Fatalf("Page 1 search failed: %v", err) } @@ -1382,7 +1385,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) { t.Fatal("Expected results on page 1") } - gistIDs2, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 2) + gistIDs2, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 2) if err != nil { t.Fatalf("Page 2 search failed: %v", err) } @@ -1398,7 +1401,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) { // Test 2: Page size - verify results per page (page size = 10) t.Run("PageSizeVerification", func(t *testing.T) { - gistIDs1, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 1) + gistIDs1, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 1) if err != nil { t.Fatalf("Page 1 search failed: %v", err) } @@ -1410,11 +1413,11 @@ func testIndexerPagination(t *testing.T, indexer Indexer) { // Test 3: Total count consistency across pages t.Run("TotalCountConsistency", func(t *testing.T) { - _, total1, _, err := indexer.Search("", SearchGistMetadata{}, 1, 1) + _, total1, _, err := indexer.Search(SearchGistMetadata{}, 1, 1) if err != nil { t.Fatalf("Page 1 search failed: %v", err) } - _, total2, _, err := indexer.Search("", SearchGistMetadata{}, 1, 2) + _, total2, _, err := indexer.Search(SearchGistMetadata{}, 1, 2) if err != nil { t.Fatalf("Page 2 search failed: %v", err) } @@ -1429,7 +1432,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) { // Test 4: Out of bounds page t.Run("OutOfBoundsPage", func(t *testing.T) { // Page 100 is way beyond 334 results with page size 10 - gistIDs, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 100) + gistIDs, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 100) if err != nil { t.Fatalf("Out of bounds page search failed: %v", err) } @@ -1440,7 +1443,8 @@ func testIndexerPagination(t *testing.T, indexer Indexer) { // Test 5: Empty results pagination t.Run("EmptyResultsPagination", func(t *testing.T) { - gistIDs, total, _, err := indexer.Search("nonexistentquery", SearchGistMetadata{}, 1, 1) + metadata := SearchGistMetadata{All: "nonexistentquery"} + gistIDs, total, _, err := indexer.Search(metadata, 1, 1) if err != nil { t.Fatalf("Empty search failed: %v", err) } @@ -1454,11 +1458,11 @@ func testIndexerPagination(t *testing.T, indexer Indexer) { // Test 6: No duplicate IDs across pages (accounting for +1 overlap for hasMore indicator) t.Run("NoDuplicateIDs", func(t *testing.T) { - gistIDs1, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 1) + gistIDs1, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 1) if err != nil { t.Fatalf("Page 1 search failed: %v", err) } - gistIDs2, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 2) + gistIDs2, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 2) if err != nil { t.Fatalf("Page 2 search failed: %v", err) } @@ -1489,7 +1493,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) { t.Run("PaginationWithFilters", func(t *testing.T) { // Filter by alice's username - should get 334 gists metadata := SearchGistMetadata{Username: "alice"} - gistIDs1, total, _, err := indexer.Search("", metadata, 1, 1) + gistIDs1, total, _, err := indexer.Search(metadata, 1, 1) if err != nil { t.Fatalf("Filtered page 1 search failed: %v", err) } @@ -1500,7 +1504,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) { t.Error("Expected results on filtered page 1") } - gistIDs2, _, _, err := indexer.Search("", metadata, 1, 2) + gistIDs2, _, _, err := indexer.Search(metadata, 1, 2) if err != nil { t.Fatalf("Filtered page 2 search failed: %v", err) } @@ -1518,7 +1522,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) { t.Run("LastPageVerification", func(t *testing.T) { // With 334 results and page size 10, page 34 should have 4 results // Let's just verify the last page has some results - gistIDs34, total, _, err := indexer.Search("", SearchGistMetadata{}, 1, 34) + gistIDs34, total, _, err := indexer.Search(SearchGistMetadata{}, 1, 34) if err != nil { t.Fatalf("Last page search failed: %v", err) } @@ -1530,7 +1534,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) { } // Page 35 should be empty - gistIDs35, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 35) + gistIDs35, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 35) if err != nil { t.Fatalf("Beyond last page search failed: %v", err) } @@ -1541,15 +1545,15 @@ func testIndexerPagination(t *testing.T, indexer Indexer) { // Test 9: Multiple pages have different results t.Run("MultiplePagesDifferent", func(t *testing.T) { - gistIDs1, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 1) + gistIDs1, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 1) if err != nil { t.Fatalf("Page 1 search failed: %v", err) } - gistIDs10, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 10) + gistIDs10, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 10) if err != nil { t.Fatalf("Page 10 search failed: %v", err) } - gistIDs20, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 20) + gistIDs20, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 20) if err != nil { t.Fatalf("Page 20 search failed: %v", err) } @@ -1568,7 +1572,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) { // Test 10: Pagination with different users (visibility filtering) t.Run("PaginationWithVisibility", func(t *testing.T) { // User 2 (bob) sees 667 gists (334 public alice + 333 own private) - gistIDs1Bob, totalBob, _, err := indexer.Search("", SearchGistMetadata{}, 2, 1) + gistIDs1Bob, totalBob, _, err := indexer.Search(SearchGistMetadata{}, 2, 1) if err != nil { t.Fatalf("Bob page 1 search failed: %v", err) } @@ -1580,7 +1584,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) { } // User 1 (alice) sees 334 gists - _, totalAlice, _, err := indexer.Search("", SearchGistMetadata{}, 1, 1) + _, totalAlice, _, err := indexer.Search(SearchGistMetadata{}, 1, 1) if err != nil { t.Fatalf("Alice page 1 search failed: %v", err) } diff --git a/internal/index/meilisearch.go b/internal/index/meilisearch.go index 9bef5b3..e71459f 100644 --- a/internal/index/meilisearch.go +++ b/internal/index/meilisearch.go @@ -9,6 +9,7 @@ import ( "github.com/meilisearch/meilisearch-go" "github.com/rs/zerolog/log" + "github.com/thomiceli/opengist/internal/config" ) type MeiliIndexer struct { @@ -63,9 +64,9 @@ func (i *MeiliIndexer) open() (meilisearch.IndexManager, error) { } _, _ = i.client.Index(i.indexName).UpdateSettings(&meilisearch.Settings{ - FilterableAttributes: []string{"GistID", "UserID", "Visibility", "Username", "Title", "Filenames", "Extensions", "Languages", "Topics"}, + FilterableAttributes: []string{"GistID", "UserID", "Visibility", "Username", "Title", "Description", "Filenames", "Extensions", "Languages", "Topics"}, DisplayedAttributes: []string{"GistID"}, - SearchableAttributes: []string{"Content", "Username", "Title", "Filenames", "Extensions", "Languages", "Topics"}, + SearchableAttributes: []string{"Content", "Username", "Title", "Description", "Filenames", "Extensions", "Languages", "Topics"}, RankingRules: []string{"words"}, }) @@ -109,7 +110,7 @@ func (i *MeiliIndexer) Remove(gistID uint) error { return err } -func (i *MeiliIndexer) Search(queryStr string, queryMetadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) { +func (i *MeiliIndexer) Search(queryMetadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) { searchRequest := &meilisearch.SearchRequest{ Offset: int64((page - 1) * 10), Limit: 11, @@ -128,6 +129,7 @@ func (i *MeiliIndexer) Search(queryStr string, queryMetadata SearchGistMetadata, } addFilter("Username", queryMetadata.Username) addFilter("Title", queryMetadata.Title) + addFilter("Description", queryMetadata.Description) addFilter("Filenames", queryMetadata.Filename) addFilter("Extensions", queryMetadata.Extension) addFilter("Languages", queryMetadata.Language) @@ -137,7 +139,29 @@ func (i *MeiliIndexer) Search(queryStr string, queryMetadata SearchGistMetadata, searchRequest.Filter = strings.Join(filters, " AND ") } - response, err := (*atomicIndexer.Load()).(*MeiliIndexer).index.Search(queryStr, searchRequest) + // build query string from provided metadata. Prefer `All`, then `Default`, fall back to `Content`. + query := queryMetadata.All + if query == "" && queryMetadata.Default != "" { + query = queryMetadata.Default + var fields []string + for _, f := range strings.Split(config.C.SearchDefault, ",") { + f = strings.TrimSpace(f) + if f == "all" { + fields = AllSearchFields + break + } + if indexField, ok := SearchFieldMap[f]; ok { + fields = append(fields, indexField) + } + } + if len(fields) > 0 { + searchRequest.AttributesToSearchOn = fields + } + } else if query == "" { + query = queryMetadata.Content + } + + response, err := (*atomicIndexer.Load()).(*MeiliIndexer).index.Search(query, searchRequest) if err != nil { log.Error().Err(err).Msg("Failed to search Meilisearch index") return nil, 0, nil, err diff --git a/internal/web/handlers/gist/all.go b/internal/web/handlers/gist/all.go index 6cc4d90..d919e24 100644 --- a/internal/web/handlers/gist/all.go +++ b/internal/web/handlers/gist/all.go @@ -164,6 +164,18 @@ func AllGists(ctx *context.Context) error { return ctx.Html("all.html") } +// Search handles the search page for gists. +// +// It takes a query parameter "q" which is a search query in the format: +// "user:username title:title description:description filename:filename language:language topic:topic" +// +// It also takes a page parameter "page" which is the page number to display. +// +// It returns an error if the search query is invalid or if the page number is invalid. +// +// It returns the search results as a list of rendered gists, along with the total number of results, the languages found, and the search query. +// +// The search results are paginated, with 10 results per page. func Search(ctx *context.Context) error { var err error @@ -171,7 +183,7 @@ func Search(ctx *context.Context) error { Query: ctx.QueryParam("q"), } - content, meta := handlers.ParseSearchQueryStr(ctx.QueryParam("q")) + metadata := handlers.ParseSearchQueryStr(ctx.QueryParam("q")) pageInt := handlers.GetPage(ctx) var currentUserId uint @@ -182,14 +194,18 @@ func Search(ctx *context.Context) error { currentUserId = 0 } - gistsIds, nbHits, langs, err := index.SearchGists(content, index.SearchGistMetadata{ - Username: meta["user"], - Title: meta["title"], - Filename: meta["filename"], - Extension: meta["extension"], - Language: meta["language"], - Topic: meta["topic"], - All: meta["all"], + // Search gists in the index and fetch the gists IDs from the database + gistsIds, nbHits, langs, err := index.SearchGists(index.SearchGistMetadata{ + Username: metadata["user"], + Title: metadata["title"], + Description: metadata["description"], + Filename: metadata["filename"], + Extension: metadata["extension"], + Language: metadata["language"], + Topic: metadata["topic"], + Content: metadata["content"], + All: metadata["all"], + Default: metadata["default"], }, currentUserId, pageInt) if err != nil { return ctx.ErrorRes(500, "Error searching gists", err) diff --git a/internal/web/handlers/util.go b/internal/web/handlers/util.go index b7ba494..bd2f2cc 100644 --- a/internal/web/handlers/util.go +++ b/internal/web/handlers/util.go @@ -8,7 +8,7 @@ import ( "strings" "github.com/gorilla/schema" - + "github.com/rs/zerolog/log" "github.com/thomiceli/opengist/internal/web/context" ) @@ -119,10 +119,16 @@ func Paginate[T any](ctx *context.Context, data []*T, pageInt int, perPage int, return nil } -func ParseSearchQueryStr(query string) (string, map[string]string) { +// ParseSearchQueryStr parses a search query string and returns a map of metadata. +// The query string is split into words and each word is checked if it contains a colon (:). +// If a word contains a colon, it is split into a key-value pair and added to the metadata map. +// If a word does not contain a colon, it is added to an "all" key in the metadata map. +// The "all" key is used to search all fields in the index. +// The function returns the metadata map. +func ParseSearchQueryStr(query string) map[string]string { words := strings.Fields(query) metadata := make(map[string]string) - var contentBuilder strings.Builder + var allFieldsBuilder strings.Builder for _, word := range words { if strings.Contains(word, ":") { @@ -133,10 +139,18 @@ func ParseSearchQueryStr(query string) (string, map[string]string) { metadata[key] = value } } else { - contentBuilder.WriteString(word + " ") + // Add to content search by default + allFieldsBuilder.WriteString(word + " ") } } - content := strings.TrimSpace(contentBuilder.String()) - return content, metadata + // Set the default search field + allContent := strings.TrimSpace(allFieldsBuilder.String()) + if allContent != "" { + metadata["default"] = allContent + } + + log.Debug().Msgf("Metadata: %v", metadata) + + return metadata } diff --git a/internal/web/server/renderer.go b/internal/web/server/renderer.go index ef7915e..5d77381 100644 --- a/internal/web/server/renderer.go +++ b/internal/web/server/renderer.go @@ -147,7 +147,10 @@ func (s *Server) setFuncMap() { return dict, nil }, "addMetadataToSearchQuery": func(input, key, value string) string { - content, metadata := handlers.ParseSearchQueryStr(input) + metadata := handlers.ParseSearchQueryStr(input) + // extract free-text content (stored under "all") and remove it from metadata + content := metadata["all"] + delete(metadata, "all") metadata[key] = value diff --git a/templates/base/base_header.html b/templates/base/base_header.html index 791fcbe..176d8dd 100644 --- a/templates/base/base_header.html +++ b/templates/base/base_header.html @@ -116,10 +116,12 @@

user:thomas {{ .locale.Tr "gist.search.help.user" }}

title:mygist {{ .locale.Tr "gist.search.help.title" }}

+

description:sync {{ .locale.Tr "gist.search.help.description" }}

filename:myfile.txt {{ .locale.Tr "gist.search.help.filename" }}

extension:yml {{ .locale.Tr "gist.search.help.extension" }}

language:go {{ .locale.Tr "gist.search.help.language" }}

topic:homelab {{ .locale.Tr "gist.search.help.topic" }}

+

all:systemctl {{ .locale.Tr "gist.search.help.all" }}