feat: search all fields (#622)

*  feat(search): search all feature

- add Description field to Gist struct and index it
- extend SearchGistMetadata with Description and Content
- update Bleve and Meilisearch to index and search Description
- modify ParseSearchQueryStr to parse description: and content: keywords
- update templates and i18n for new search options

* Fix test

* Set content by default

Signed-off-by: Thomas Miceli <tho.miceli@gmail.com>

* Config to define default searchable fields

Signed-off-by: Thomas Miceli <tho.miceli@gmail.com>

---------

Signed-off-by: Thomas Miceli <tho.miceli@gmail.com>
Co-authored-by: Thomas Miceli <tho.miceli@gmail.com>
This commit is contained in:
Webysther Sperandio
2026-03-11 17:55:23 +01:00
committed by GitHub
parent 5ad01a3304
commit 279da52899
15 changed files with 338 additions and 187 deletions

View File

@@ -32,6 +32,10 @@ index.meili.host:
# Set the API key for the Meiliseach server
index.meili.api-key:
# Set the default search fields. Can contain multiple fields (e.g., `content,username`).
# Fields: content,user,title,description,filename,extension,language,topic. Default: content
search.default: content
# Default branch name used by Opengist when initializing Git repositories.
# If not set, uses the Git default branch name. See https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup#_new_default_branch
git.default-branch:

View File

@@ -15,6 +15,7 @@ aside: false
| index | OG_INDEX | `bleve` | Define the code indexer (either `bleve`, `meilisearch`, or empty for no index). |
| index.meili.host | OG_MEILI_HOST | none | Set the host for the Meiliseach server. |
| index.meili.api-key | OG_MEILI_API_KEY | none | Set the API key for the Meiliseach server. |
| search.default | OG_SEARCH_DEFAULT | `content` | Set the default search fields. Can contain multiple fields (e.g., `content,username`). Fields: `content,user,title,description,filename,extension,language,topic`. |
| git.default-branch | OG_GIT_DEFAULT_BRANCH | none | Default branch name used by Opengist when initializing Git repositories. If not set, uses the Git default branch name. More info [here](https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup#_new_default_branch) |
| sqlite.journal-mode | OG_SQLITE_JOURNAL_MODE | `WAL` | Set the journal mode for SQLite. More info [here](https://www.sqlite.org/pragma.html#pragma_journal_mode) |
| http.host | OG_HTTP_HOST | `0.0.0.0` | The host on which the HTTP server should bind. Use an IP address for network binding. Use a path for Unix socket binding (e.g. /run/opengist.sock) |

View File

@@ -38,11 +38,12 @@ type config struct {
DBUri string `yaml:"db-uri" env:"OG_DB_URI"`
DBFilename string `yaml:"db-filename" env:"OG_DB_FILENAME"` // deprecated
IndexEnabled bool `yaml:"index.enabled" env:"OG_INDEX_ENABLED"` // deprecated
Index string `yaml:"index" env:"OG_INDEX"`
BleveDirname string `yaml:"index.dirname" env:"OG_INDEX_DIRNAME"` // deprecated
MeiliHost string `yaml:"index.meili.host" env:"OG_MEILI_HOST"`
MeiliAPIKey string `yaml:"index.meili.api-key" env:"OG_MEILI_API_KEY"`
IndexEnabled bool `yaml:"index.enabled" env:"OG_INDEX_ENABLED"` // deprecated
Index string `yaml:"index" env:"OG_INDEX"`
BleveDirname string `yaml:"index.dirname" env:"OG_INDEX_DIRNAME"` // deprecated
MeiliHost string `yaml:"index.meili.host" env:"OG_MEILI_HOST"`
MeiliAPIKey string `yaml:"index.meili.api-key" env:"OG_MEILI_API_KEY"`
SearchDefault string `yaml:"search.default" env:"OG_SEARCH_DEFAULT"`
GitDefaultBranch string `yaml:"git.default-branch" env:"OG_GIT_DEFAULT_BRANCH"`
@@ -111,6 +112,7 @@ func configWithDefaults() (*config, error) {
c.OpengistHome = ""
c.DBUri = "opengist.db"
c.Index = "bleve"
c.SearchDefault = "content"
c.SqliteJournalMode = "WAL"

View File

@@ -817,18 +817,19 @@ func (gist *Gist) ToIndexedGist() (*index.Gist, error) {
}
indexedGist := &index.Gist{
GistID: gist.ID,
UserID: gist.UserID,
Visibility: gist.Private.Uint(),
Username: gist.User.Username,
Title: gist.Title,
Content: wholeContent,
Filenames: fileNames,
Extensions: exts,
Languages: langs,
Topics: topics,
CreatedAt: gist.CreatedAt,
UpdatedAt: gist.UpdatedAt,
GistID: gist.ID,
UserID: gist.UserID,
Visibility: gist.Private.Uint(),
Username: gist.User.Username,
Description: gist.Description,
Title: gist.Title,
Content: wholeContent,
Filenames: fileNames,
Extensions: exts,
Languages: langs,
Topics: topics,
CreatedAt: gist.CreatedAt,
UpdatedAt: gist.UpdatedAt,
}
return indexedGist, nil

View File

@@ -88,10 +88,12 @@ gist.search.found: gists found
gist.search.no-results: No gists found
gist.search.help.user: gists created by user
gist.search.help.title: gists with given title
gist.search.help.description: gists with given description
gist.search.help.filename: gists having files with given name
gist.search.help.extension: gists having files with given extension
gist.search.help.language: gists having files with given language
gist.search.help.topic: gists with given topic
gist.search.help.all: search all fields
gist.search.placeholder.title: Title
gist.search.placeholder.visibility: Visibility
gist.search.placeholder.public: Public

View File

@@ -5,6 +5,7 @@ import (
"fmt"
"os"
"strconv"
"strings"
"github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
@@ -14,6 +15,7 @@ import (
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
"github.com/blevesearch/bleve/v2/search/query"
"github.com/rs/zerolog/log"
"github.com/thomiceli/opengist/internal/config"
)
type BleveIndexer struct {
@@ -116,18 +118,60 @@ func (i *BleveIndexer) Remove(gistID uint) error {
return (*atomicIndexer.Load()).(*BleveIndexer).index.Delete(strconv.Itoa(int(gistID)))
}
func (i *BleveIndexer) Search(queryStr string, queryMetadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
// Search returns a list of Gist IDs that match the given search metadata.
// The method returns an error if any.
//
// The queryMetadata parameter is used to filter the search results.
// For example, passing a non-empty Username will search for gists whose
// username matches the given string.
//
// If the "All" field in queryMetadata is non-empty, the method will
// search across all metadata fields with OR logic. Otherwise, the method
// will add each metadata field with AND logic.
//
// The page parameter is used to paginate the search results.
// The method returns the total number of search results in the second return
// value.
//
// The third return value is a map of language counts for the search results.
// The language counts are computed by asking Bleve to return the top 10
// facets for the "Languages" field.
func (i *BleveIndexer) Search(metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
var err error
var indexerQuery query.Query
if queryStr != "" {
// Use match query with fuzzy matching for more flexible content search
contentQuery := bleve.NewMatchQuery(queryStr)
contentQuery.SetField("Content")
contentQuery.SetFuzziness(2)
indexerQuery = contentQuery
} else {
contentQuery := bleve.NewMatchAllQuery()
indexerQuery = contentQuery
var indexerQuery query.Query = bleve.NewMatchAllQuery()
// Query factory
factoryQuery := func(field, value string) query.Query {
query := bleve.NewMatchPhraseQuery(value)
query.SetField(field)
return query
}
// Exact search
addQuery := func(field, value string) {
if value != "" && value != "." {
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, factoryQuery(field, value))
}
}
// Exact+fuzzy query factory: exact match is boosted so it ranks above fuzzy-only matches
factoryFuzzyQuery := func(field, value string) query.Query {
exact := bleve.NewMatchPhraseQuery(value)
exact.SetField(field)
exact.SetBoost(2.0)
fuzzy := bleve.NewMatchQuery(value)
fuzzy.SetField(field)
fuzzy.SetFuzziness(2)
return bleve.NewDisjunctionQuery(exact, fuzzy)
}
// Exact+fuzzy search
addFuzzy := func(field, value string) {
if value != "" && value != "." {
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, factoryFuzzyQuery(field, value))
}
}
// Visibility filtering: show public gists (Visibility=0) OR user's own gists
@@ -143,48 +187,58 @@ func (i *BleveIndexer) Search(queryStr string, queryMetadata SearchGistMetadata,
accessQuery := bleve.NewDisjunctionQuery(publicQuery, userIdQuery)
indexerQuery = bleve.NewConjunctionQuery(accessQuery, indexerQuery)
buildFieldQuery := func(field, value string) query.Query {
switch field {
case "Title", "Description", "Filenames", "Content":
return factoryFuzzyQuery(field, value)
case "Extensions":
return factoryQuery(field, "."+value)
default: // Username, Languages, Topics
return factoryQuery(field, value)
}
}
// Handle "All" field - search across all metadata fields with OR logic
if queryMetadata.All != "" {
allQueries := make([]query.Query, 0)
// Create match phrase queries for each field
fields := []struct {
field string
value string
}{
{"Username", queryMetadata.All},
{"Title", queryMetadata.All},
{"Extensions", "." + queryMetadata.All},
{"Filenames", queryMetadata.All},
{"Languages", queryMetadata.All},
{"Topics", queryMetadata.All},
if metadata.All != "" {
allQueries := make([]query.Query, 0, len(AllSearchFields))
for _, field := range AllSearchFields {
allQueries = append(allQueries, buildFieldQuery(field, metadata.All))
}
for _, f := range fields {
q := bleve.NewMatchPhraseQuery(f.value)
q.FieldVal = f.field
allQueries = append(allQueries, q)
}
// Combine all field queries with OR (disjunction)
allDisjunction := bleve.NewDisjunctionQuery(allQueries...)
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, allDisjunction)
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, bleve.NewDisjunctionQuery(allQueries...))
} else {
// Original behavior: add each metadata field with AND logic
addQuery := func(field, value string) {
if value != "" && value != "." {
q := bleve.NewMatchPhraseQuery(value)
q.FieldVal = field
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, q)
addQuery("Username", metadata.Username)
addFuzzy("Title", metadata.Title)
addFuzzy("Description", metadata.Description)
addQuery("Extensions", "."+metadata.Extension)
addFuzzy("Filenames", metadata.Filename)
addQuery("Languages", metadata.Language)
addQuery("Topics", metadata.Topic)
addFuzzy("Content", metadata.Content)
// Handle default search fields from config with OR logic
if metadata.Default != "" {
var fields []string
for _, f := range strings.Split(config.C.SearchDefault, ",") {
f = strings.TrimSpace(f)
if f == "all" {
fields = AllSearchFields
break
}
if indexField, ok := SearchFieldMap[f]; ok {
fields = append(fields, indexField)
}
}
if len(fields) == 1 {
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, buildFieldQuery(fields[0], metadata.Default))
} else if len(fields) > 1 {
defaultQueries := make([]query.Query, 0, len(fields))
for _, field := range fields {
defaultQueries = append(defaultQueries, buildFieldQuery(field, metadata.Default))
}
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, bleve.NewDisjunctionQuery(defaultQueries...))
}
}
addQuery("Username", queryMetadata.Username)
addQuery("Title", queryMetadata.Title)
addQuery("Extensions", "."+queryMetadata.Extension)
addQuery("Filenames", queryMetadata.Filename)
addQuery("Languages", queryMetadata.Language)
addQuery("Topics", queryMetadata.Topic)
}
languageFacet := bleve.NewFacetRequest("Languages", 10)
@@ -197,6 +251,8 @@ func (i *BleveIndexer) Search(queryStr string, queryMetadata SearchGistMetadata,
s.Fields = []string{"GistID"}
s.IncludeLocations = false
log.Debug().Interface("searchRequest", s).Msg("Bleve search request")
results, err := (*atomicIndexer.Load()).(*BleveIndexer).index.Search(s)
if err != nil {
return nil, 0, nil, err

View File

@@ -119,18 +119,19 @@ func TestBleveIndexerUnicodeSearch(t *testing.T) {
// Add a gist with Unicode content
gist := &Gist{
GistID: 100,
UserID: 100,
Visibility: 0,
Username: "testuser",
Title: "Unicode Test",
Content: "Hello world with unicode characters: café résumé naïve",
Filenames: []string{"test.txt"},
Extensions: []string{".txt"},
Languages: []string{"Text"},
Topics: []string{"unicode"},
CreatedAt: 1234567890,
UpdatedAt: 1234567890,
GistID: 100,
UserID: 100,
Visibility: 0,
Username: "testuser",
Title: "Unicode Test",
Description: "Descrition with Unicode characters: Café résumé naive",
Content: "Hello world with unicode characters: café résumé naïve",
Filenames: []string{"test.txt"},
Extensions: []string{".txt"},
Languages: []string{"Text"},
Topics: []string{"unicode"},
CreatedAt: 1234567890,
UpdatedAt: 1234567890,
}
err := indexer.Add(gist)
@@ -139,7 +140,7 @@ func TestBleveIndexerUnicodeSearch(t *testing.T) {
}
// Search for unicode content
gistIDs, total, _, err := indexer.Search("café", SearchGistMetadata{}, 100, 1)
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "café"}, 100, 1)
if err != nil {
t.Fatalf("Search failed: %v", err)
}

View File

@@ -1,26 +1,43 @@
package index
var AllSearchFields = []string{"Username", "Title", "Description", "Filenames", "Extensions", "Languages", "Topics", "Content"}
var SearchFieldMap = map[string]string{
"user": "Username",
"title": "Title",
"description": "Description",
"filename": "Filenames",
"extension": "Extensions",
"language": "Languages",
"topic": "Topics",
"content": "Content",
}
type Gist struct {
GistID uint
UserID uint
Visibility uint
Username string
Title string
Content string
Filenames []string
Extensions []string
Languages []string
Topics []string
CreatedAt int64
UpdatedAt int64
GistID uint
UserID uint
Visibility uint
Username string
Description string
Title string
Content string
Filenames []string
Extensions []string
Languages []string
Topics []string
CreatedAt int64
UpdatedAt int64
}
type SearchGistMetadata struct {
Username string
Title string
Filename string
Extension string
Language string
Topic string
All string
Username string
Title string
Description string
Content string
Filename string
Extension string
Language string
Topic string
All string
Default string
}

View File

@@ -17,7 +17,7 @@ type Indexer interface {
Reset() error
Add(gist *Gist) error
Remove(gistID uint) error
Search(query string, metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error)
Search(metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error)
}
type IndexerType string
@@ -125,7 +125,11 @@ func RemoveFromIndex(gistID uint) error {
return (*idx).Remove(gistID)
}
func SearchGists(query string, metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
// SearchGists returns a list of Gist IDs that match the given search metadata.
// If the indexer is not enabled, it returns nil, 0, nil, nil.
// If the indexer is not initialized, it returns nil, 0, nil, fmt.Errorf("indexer is not initialized").
// The function returns an error if any.
func SearchGists(metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
if !IndexEnabled() {
return nil, 0, nil, nil
}
@@ -135,7 +139,7 @@ func SearchGists(query string, metadata SearchGistMetadata, userId uint, page in
return nil, 0, nil, fmt.Errorf("indexer is not initialized")
}
return (*idx).Search(query, metadata, userId, page)
return (*idx).Search(metadata, userId, page)
}
func DepreactionIndexDirname() {

View File

@@ -71,7 +71,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) {
}
// Verify gist is searchable
gistIDs, total, _, err := indexer.Search("test gist", SearchGistMetadata{}, 11, 1)
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "test gist"}, 11, 1)
if err != nil {
t.Fatalf("Search failed: %v", err)
}
@@ -114,7 +114,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) {
// Search by Rust language
metadata := SearchGistMetadata{Language: "Rust"}
gistIDs, total, _, err := indexer.Search("", metadata, 11, 1)
gistIDs, total, _, err := indexer.Search(metadata, 11, 1)
if err != nil {
t.Fatalf("Search by Rust language failed: %v", err)
}
@@ -156,7 +156,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) {
}
// Search for unicode content
_, total, _, err := indexer.Search("café", SearchGistMetadata{}, 11, 1)
_, total, _, err := indexer.Search(SearchGistMetadata{All: "café"}, 11, 1)
if err != nil {
t.Fatalf("Search for unicode failed: %v", err)
}
@@ -189,7 +189,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) {
}
// User 11 should see their own private gist
gistIDs, total, _, err := indexer.Search("private gist", SearchGistMetadata{}, 11, 1)
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "private gist"}, 11, 1)
if err != nil {
t.Fatalf("Search for private gist as owner failed: %v", err)
}
@@ -205,7 +205,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) {
}
// User 999 should NOT see user 11's private gist
gistIDs2, _, _, err := indexer.Search("private gist", SearchGistMetadata{}, 999, 1)
gistIDs2, _, _, err := indexer.Search(SearchGistMetadata{All: "private gist"}, 999, 1)
if err != nil {
t.Fatalf("Search for private gist as other user failed: %v", err)
}
@@ -239,7 +239,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) {
}
// Should still be searchable by content
_, total, _, err := indexer.Search("Minimal", SearchGistMetadata{}, 11, 1)
_, total, _, err := indexer.Search(SearchGistMetadata{All: "Minimal"}, 11, 1)
if err != nil {
t.Fatalf("Search for minimal gist failed: %v", err)
}
@@ -292,7 +292,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) {
}
// Search should find updated content, not original
gistIDs, total, _, err := indexer.Search("new information", SearchGistMetadata{}, 11, 1)
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "new information"}, 11, 1)
if err != nil {
t.Fatalf("Search for updated content failed: %v", err)
}
@@ -308,7 +308,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) {
}
// Old content should not be found
gistIDsOld, _, _, _ := indexer.Search("Original", SearchGistMetadata{}, 11, 1)
gistIDsOld, _, _, _ := indexer.Search(SearchGistMetadata{All: "Original"}, 11, 1)
for _, id := range gistIDsOld {
if id == 1006 {
t.Error("Should not find gist by old content after update")
@@ -340,7 +340,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) {
// Search by username
metadata := SearchGistMetadata{Username: "newuser"}
gistIDs, total, _, err := indexer.Search("", metadata, 12, 1)
gistIDs, total, _, err := indexer.Search(metadata, 12, 1)
if err != nil {
t.Fatalf("Search by username failed: %v", err)
}
@@ -383,7 +383,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) {
// Search by one of the topics
metadata := SearchGistMetadata{Topic: "microservices"}
gistIDs, total, _, err := indexer.Search("", metadata, 11, 1)
gistIDs, total, _, err := indexer.Search(metadata, 11, 1)
if err != nil {
t.Fatalf("Search by topic failed: %v", err)
}
@@ -427,7 +427,7 @@ func testIndexerAddGist(t *testing.T, indexer Indexer) {
}
// Search for content from the middle
gistIDs, total, _, err := indexer.Search("Line 500", SearchGistMetadata{}, 11, 1)
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "Line 500"}, 11, 1)
if err != nil {
t.Fatalf("Search in long content failed: %v", err)
}
@@ -451,7 +451,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
// Test 1: Search by content - all init gists have "searchable content"
t.Run("SearchByContent", func(t *testing.T) {
gistIDs, total, languageCounts, err := indexer.Search("searchable", SearchGistMetadata{}, 1, 1)
gistIDs, total, languageCounts, err := indexer.Search(SearchGistMetadata{All: "searchable"}, 1, 1)
if err != nil {
t.Fatalf("Search failed: %v", err)
}
@@ -475,7 +475,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
// Test 2: Search by specific language - Go
t.Run("SearchByLanguage", func(t *testing.T) {
metadata := SearchGistMetadata{Language: "Go"}
_, total, _, err := indexer.Search("", metadata, 1, 1)
_, total, _, err := indexer.Search(metadata, 1, 1)
if err != nil {
t.Fatalf("Search by language failed: %v", err)
}
@@ -489,7 +489,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
// Test 3: Search by specific username - alice
t.Run("SearchByUsername", func(t *testing.T) {
metadata := SearchGistMetadata{Username: "alice"}
_, total, _, err := indexer.Search("", metadata, 1, 1)
_, total, _, err := indexer.Search(metadata, 1, 1)
if err != nil {
t.Fatalf("Search by username failed: %v", err)
}
@@ -503,7 +503,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
// Test 4: Search by extension - Python (bob's private files)
t.Run("SearchByExtension", func(t *testing.T) {
metadata := SearchGistMetadata{Extension: "py"}
_, total, _, err := indexer.Search("", metadata, 1, 1)
_, total, _, err := indexer.Search(metadata, 1, 1)
if err != nil {
t.Fatalf("Search by extension failed: %v", err)
}
@@ -518,7 +518,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
// Test 5: Search by topic - algorithms
t.Run("SearchByTopic", func(t *testing.T) {
metadata := SearchGistMetadata{Topic: "algorithms"}
_, total, _, err := indexer.Search("", metadata, 1, 1)
_, total, _, err := indexer.Search(metadata, 1, 1)
if err != nil {
t.Fatalf("Search by topic failed: %v", err)
}
@@ -535,7 +535,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
Language: "Go",
Username: "alice",
}
_, total, _, err := indexer.Search("", metadata, 1, 1)
_, total, _, err := indexer.Search(metadata, 1, 1)
if err != nil {
t.Fatalf("Search with combined filters failed: %v", err)
}
@@ -548,7 +548,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
// Test 7: Search with no results
t.Run("SearchNoResults", func(t *testing.T) {
gistIDs, total, _, err := indexer.Search("nonexistentquery", SearchGistMetadata{}, 1, 1)
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "nonexistentquery"}, 1, 1)
if err != nil {
t.Fatalf("Search with no results failed: %v", err)
}
@@ -562,7 +562,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
// Test 8: Empty query returns all accessible gists
t.Run("SearchEmptyQuery", func(t *testing.T) {
gistIDs, total, languageCounts, err := indexer.Search("", SearchGistMetadata{}, 1, 1)
gistIDs, total, languageCounts, err := indexer.Search(SearchGistMetadata{}, 1, 1)
if err != nil {
t.Fatalf("Empty search failed: %v", err)
}
@@ -586,7 +586,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
t.Run("SearchPagination", func(t *testing.T) {
// As user 1, we have 334 gists total
// Page 1
gistIDs1, total, _, err := indexer.Search("searchable", SearchGistMetadata{}, 1, 1)
gistIDs1, total, _, err := indexer.Search(SearchGistMetadata{All: "searchable"}, 1, 1)
if err != nil {
t.Fatalf("Page 1 search failed: %v", err)
}
@@ -598,7 +598,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
}
// Page 2
gistIDs2, _, _, err := indexer.Search("searchable", SearchGistMetadata{}, 1, 2)
gistIDs2, _, _, err := indexer.Search(SearchGistMetadata{All: "searchable"}, 1, 2)
if err != nil {
t.Fatalf("Page 2 search failed: %v", err)
}
@@ -619,7 +619,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
// Search as user 2 (bob)
// bob has 333 gists at i=1,4,7,... with visibility=1 (private)
// As user 2, bob sees: alice's 334 public gists + bob's own 333 gists = 667 total
_, total, _, err := indexer.Search("", SearchGistMetadata{}, 2, 1)
_, total, _, err := indexer.Search(SearchGistMetadata{}, 2, 1)
if err != nil {
t.Fatalf("Search as user 2 failed: %v", err)
}
@@ -628,7 +628,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
}
// Search as non-existent user (should only see public gists)
_, totalPublic, _, err := indexer.Search("", SearchGistMetadata{}, 999, 1)
_, totalPublic, _, err := indexer.Search(SearchGistMetadata{}, 999, 1)
if err != nil {
t.Fatalf("Search as user 999 failed: %v", err)
}
@@ -645,7 +645,7 @@ func testIndexerSearchBasic(t *testing.T, indexer Indexer) {
// Test 11: Language facets validation
t.Run("LanguageFacets", func(t *testing.T) {
_, _, languageCounts, err := indexer.Search("", SearchGistMetadata{}, 1, 1)
_, _, languageCounts, err := indexer.Search(SearchGistMetadata{}, 1, 1)
if err != nil {
t.Fatalf("Search for facets failed: %v", err)
}
@@ -755,7 +755,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) {
// Test 1: All field matches username
t.Run("AllFieldMatchesUsername", func(t *testing.T) {
metadata := SearchGistMetadata{All: "testuser_unique"}
gistIDs, total, _, err := indexer.Search("", metadata, 100, 1)
gistIDs, total, _, err := indexer.Search(metadata, 100, 1)
if err != nil {
t.Fatalf("All field search failed: %v", err)
}
@@ -777,7 +777,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) {
// Test 2: All field matches title
t.Run("AllFieldMatchesTitle", func(t *testing.T) {
metadata := SearchGistMetadata{All: "unique features"}
gistIDs, total, _, err := indexer.Search("", metadata, 100, 1)
gistIDs, total, _, err := indexer.Search(metadata, 100, 1)
if err != nil {
t.Fatalf("All field search failed: %v", err)
}
@@ -799,7 +799,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) {
// Test 3: All field matches language
t.Run("AllFieldMatchesLanguage", func(t *testing.T) {
metadata := SearchGistMetadata{All: "Ruby"}
gistIDs, total, _, err := indexer.Search("", metadata, 100, 1)
gistIDs, total, _, err := indexer.Search(metadata, 100, 1)
if err != nil {
t.Fatalf("All field search failed: %v", err)
}
@@ -821,7 +821,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) {
// Test 4: All field matches topic
t.Run("AllFieldMatchesTopic", func(t *testing.T) {
metadata := SearchGistMetadata{All: "unique_topic"}
gistIDs, total, _, err := indexer.Search("", metadata, 100, 1)
gistIDs, total, _, err := indexer.Search(metadata, 100, 1)
if err != nil {
t.Fatalf("All field search failed: %v", err)
}
@@ -843,7 +843,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) {
// Test 5: All field matches extension
t.Run("AllFieldMatchesExtension", func(t *testing.T) {
metadata := SearchGistMetadata{All: "sh"}
gistIDs, total, _, err := indexer.Search("", metadata, 100, 1)
gistIDs, total, _, err := indexer.Search(metadata, 100, 1)
if err != nil {
t.Fatalf("All field search failed: %v", err)
}
@@ -865,7 +865,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) {
// Test 6: All field matches filename
t.Run("AllFieldMatchesFilename", func(t *testing.T) {
metadata := SearchGistMetadata{All: "unique_file.rb"}
gistIDs, total, _, err := indexer.Search("", metadata, 100, 1)
gistIDs, total, _, err := indexer.Search(metadata, 100, 1)
if err != nil {
t.Fatalf("All field search failed: %v", err)
}
@@ -888,7 +888,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) {
t.Run("AllFieldORBehavior", func(t *testing.T) {
// "unique" appears in: username (3001), title (3002), topic (3003), filename (3004)
metadata := SearchGistMetadata{All: "unique"}
gistIDs, total, _, err := indexer.Search("", metadata, 100, 1)
gistIDs, total, _, err := indexer.Search(metadata, 100, 1)
if err != nil {
t.Fatalf("All field OR search failed: %v", err)
}
@@ -916,14 +916,14 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) {
t.Run("AllFieldVsSpecificField", func(t *testing.T) {
// Search with All field
metadataAll := SearchGistMetadata{All: "unique"}
_, totalAll, _, err := indexer.Search("", metadataAll, 100, 1)
_, totalAll, _, err := indexer.Search(metadataAll, 100, 1)
if err != nil {
t.Fatalf("All field search failed: %v", err)
}
// Search with specific username field only
metadataSpecific := SearchGistMetadata{Username: "testuser_unique"}
_, totalSpecific, _, err := indexer.Search("", metadataSpecific, 100, 1)
_, totalSpecific, _, err := indexer.Search(metadataSpecific, 100, 1)
if err != nil {
t.Fatalf("Specific field search failed: %v", err)
}
@@ -936,8 +936,8 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) {
// Test 9: All field with no matches
t.Run("AllFieldNoMatches", func(t *testing.T) {
metadata := SearchGistMetadata{All: "nonexistentvalue12345"}
gistIDs, total, _, err := indexer.Search("", metadata, 100, 1)
metadata := SearchGistMetadata{All: "nonexistentvalue"}
gistIDs, total, _, err := indexer.Search(metadata, 100, 1)
if err != nil {
t.Fatalf("All field no match search failed: %v", err)
}
@@ -957,7 +957,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) {
Username: "nonexistent", // This should be ignored
Language: "NonExistent", // This should be ignored
}
gistIDs, total, _, err := indexer.Search("", metadata, 100, 1)
gistIDs, total, _, err := indexer.Search(metadata, 100, 1)
if err != nil {
t.Fatalf("All field with other fields search failed: %v", err)
}
@@ -983,7 +983,9 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) {
// All field searches metadata, content query searches content
// Both should work together
metadata := SearchGistMetadata{All: "Ruby"}
gistIDs, total, _, err := indexer.Search("examples", metadata, 100, 1)
// Use All field for content search
metadata.All = "examples"
gistIDs, total, _, err := indexer.Search(metadata, 100, 1)
if err != nil {
t.Fatalf("All field with content query failed: %v", err)
}
@@ -1007,7 +1009,7 @@ func testIndexerAllFieldSearch(t *testing.T, indexer Indexer) {
t.Run("AllFieldCaseInsensitive", func(t *testing.T) {
// Search with different case
metadata := SearchGistMetadata{All: "RUBY"}
gistIDs, total, _, err := indexer.Search("", metadata, 100, 1)
gistIDs, total, _, err := indexer.Search(metadata, 100, 1)
if err != nil {
t.Fatalf("All field case insensitive search failed: %v", err)
}
@@ -1101,7 +1103,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) {
// Test 1: Exact match should work
t.Run("ExactMatch", func(t *testing.T) {
gistIDs, total, _, err := indexer.Search("algorithms", SearchGistMetadata{}, 100, 1)
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "algorithms"}, 100, 1)
if err != nil {
t.Fatalf("Exact match search failed: %v", err)
}
@@ -1123,7 +1125,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) {
// Test 2: 1 character typo - substitution
t.Run("OneCharSubstitution", func(t *testing.T) {
// "algoritm" instead of "algorithm" (missing 'h')
gistIDs, total, _, err := indexer.Search("algoritm", SearchGistMetadata{}, 100, 1)
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "algoritm"}, 100, 1)
if err != nil {
t.Fatalf("1-char typo search failed: %v", err)
}
@@ -1145,7 +1147,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) {
// Test 3: 1 character typo - deletion
t.Run("OneCharDeletion", func(t *testing.T) {
// "pythn" instead of "python" (missing 'o')
gistIDs, total, _, err := indexer.Search("pythn", SearchGistMetadata{}, 100, 1)
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "pythn"}, 100, 1)
if err != nil {
t.Fatalf("1-char deletion search failed: %v", err)
}
@@ -1167,7 +1169,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) {
// Test 4: 1 character typo - insertion (extra character)
t.Run("OneCharInsertion", func(t *testing.T) {
// "pythonn" instead of "python" (extra 'n')
gistIDs, total, _, err := indexer.Search("pythonn", SearchGistMetadata{}, 100, 1)
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "pythonn"}, 100, 1)
if err != nil {
t.Fatalf("1-char insertion search failed: %v", err)
}
@@ -1189,7 +1191,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) {
// Test 5: 2 character typos - should still match with fuzziness=2
t.Run("TwoCharTypos", func(t *testing.T) {
// "databse" instead of "database" (missing 'a', transposed 's')
gistIDs, total, _, err := indexer.Search("databse", SearchGistMetadata{}, 100, 1)
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "databse"}, 100, 1)
if err != nil {
t.Fatalf("2-char typo search failed: %v", err)
}
@@ -1211,7 +1213,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) {
// Test 6: 2 character typos - different word
t.Run("TwoCharTyposDifferentWord", func(t *testing.T) {
// "javasript" instead of "javascript" (missing 'c')
gistIDs, total, _, err := indexer.Search("javasript", SearchGistMetadata{}, 100, 1)
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "javasript"}, 100, 1)
if err != nil {
t.Fatalf("2-char typo search failed: %v", err)
}
@@ -1233,7 +1235,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) {
// Test 7: 3 character typos - should NOT match (beyond fuzziness=2)
t.Run("ThreeCharTyposShouldNotMatch", func(t *testing.T) {
// "algorthm" instead of "algorithm" (missing 'i', 't', 'h') - too different
gistIDs, _, _, err := indexer.Search("algorthm", SearchGistMetadata{}, 100, 1)
gistIDs, _, _, err := indexer.Search(SearchGistMetadata{All: "algorthm"}, 100, 1)
if err != nil {
t.Fatalf("3-char typo search failed: %v", err)
}
@@ -1256,7 +1258,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) {
// Test 8: Transposition (swapped characters)
t.Run("CharacterTransposition", func(t *testing.T) {
// "pyhton" instead of "python" (swapped 'ht')
gistIDs, total, _, err := indexer.Search("pyhton", SearchGistMetadata{}, 100, 1)
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "pyhton"}, 100, 1)
if err != nil {
t.Fatalf("Transposition search failed: %v", err)
}
@@ -1278,7 +1280,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) {
// Test 9: Case insensitivity with fuzzy search
t.Run("CaseInsensitiveWithFuzzy", func(t *testing.T) {
// "PYTHN" (uppercase with typo)
gistIDs, total, _, err := indexer.Search("PYTHN", SearchGistMetadata{}, 100, 1)
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "PYTHN"}, 100, 1)
if err != nil {
t.Fatalf("Case insensitive fuzzy search failed: %v", err)
}
@@ -1300,7 +1302,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) {
// Test 10: Multiple words with typos
t.Run("MultipleWordsWithTypos", func(t *testing.T) {
// "relatonal databse" instead of "relational database"
gistIDs, total, _, err := indexer.Search("relatonal databse", SearchGistMetadata{}, 100, 1)
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "relatonal databse"}, 100, 1)
if err != nil {
t.Fatalf("Multi-word fuzzy search failed: %v", err)
}
@@ -1322,7 +1324,7 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) {
// Test 11: Short words with typos (edge case)
t.Run("ShortWordsWithTypos", func(t *testing.T) {
// "SLQ" instead of "SQL" (1 char typo on short word)
gistIDs, total, _, err := indexer.Search("SLQ", SearchGistMetadata{}, 100, 1)
gistIDs, total, _, err := indexer.Search(SearchGistMetadata{All: "SLQ"}, 100, 1)
if err != nil {
t.Fatalf("Short word fuzzy search failed: %v", err)
}
@@ -1345,7 +1347,8 @@ func testIndexerFuzzySearch(t *testing.T, indexer Indexer) {
t.Run("FuzzySearchWithMetadataFilters", func(t *testing.T) {
// Search with typo AND username filter
metadata := SearchGistMetadata{Username: "fuzzytest"}
gistIDs, total, _, err := indexer.Search("algoritm", metadata, 100, 1)
metadata.All = "algoritm"
gistIDs, total, _, err := indexer.Search(metadata, 100, 1)
if err != nil {
t.Fatalf("Fuzzy search with metadata failed: %v", err)
}
@@ -1371,7 +1374,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
// Test 1: Basic pagination - pages should be different
t.Run("BasicPagination", func(t *testing.T) {
// Search as user 1 (alice) - should see 334 public gists
gistIDs1, total, _, err := indexer.Search("", SearchGistMetadata{}, 1, 1)
gistIDs1, total, _, err := indexer.Search(SearchGistMetadata{}, 1, 1)
if err != nil {
t.Fatalf("Page 1 search failed: %v", err)
}
@@ -1382,7 +1385,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
t.Fatal("Expected results on page 1")
}
gistIDs2, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 2)
gistIDs2, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 2)
if err != nil {
t.Fatalf("Page 2 search failed: %v", err)
}
@@ -1398,7 +1401,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
// Test 2: Page size - verify results per page (page size = 10)
t.Run("PageSizeVerification", func(t *testing.T) {
gistIDs1, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 1)
gistIDs1, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 1)
if err != nil {
t.Fatalf("Page 1 search failed: %v", err)
}
@@ -1410,11 +1413,11 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
// Test 3: Total count consistency across pages
t.Run("TotalCountConsistency", func(t *testing.T) {
_, total1, _, err := indexer.Search("", SearchGistMetadata{}, 1, 1)
_, total1, _, err := indexer.Search(SearchGistMetadata{}, 1, 1)
if err != nil {
t.Fatalf("Page 1 search failed: %v", err)
}
_, total2, _, err := indexer.Search("", SearchGistMetadata{}, 1, 2)
_, total2, _, err := indexer.Search(SearchGistMetadata{}, 1, 2)
if err != nil {
t.Fatalf("Page 2 search failed: %v", err)
}
@@ -1429,7 +1432,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
// Test 4: Out of bounds page
t.Run("OutOfBoundsPage", func(t *testing.T) {
// Page 100 is way beyond 334 results with page size 10
gistIDs, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 100)
gistIDs, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 100)
if err != nil {
t.Fatalf("Out of bounds page search failed: %v", err)
}
@@ -1440,7 +1443,8 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
// Test 5: Empty results pagination
t.Run("EmptyResultsPagination", func(t *testing.T) {
gistIDs, total, _, err := indexer.Search("nonexistentquery", SearchGistMetadata{}, 1, 1)
metadata := SearchGistMetadata{All: "nonexistentquery"}
gistIDs, total, _, err := indexer.Search(metadata, 1, 1)
if err != nil {
t.Fatalf("Empty search failed: %v", err)
}
@@ -1454,11 +1458,11 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
// Test 6: No duplicate IDs across pages (accounting for +1 overlap for hasMore indicator)
t.Run("NoDuplicateIDs", func(t *testing.T) {
gistIDs1, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 1)
gistIDs1, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 1)
if err != nil {
t.Fatalf("Page 1 search failed: %v", err)
}
gistIDs2, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 2)
gistIDs2, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 2)
if err != nil {
t.Fatalf("Page 2 search failed: %v", err)
}
@@ -1489,7 +1493,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
t.Run("PaginationWithFilters", func(t *testing.T) {
// Filter by alice's username - should get 334 gists
metadata := SearchGistMetadata{Username: "alice"}
gistIDs1, total, _, err := indexer.Search("", metadata, 1, 1)
gistIDs1, total, _, err := indexer.Search(metadata, 1, 1)
if err != nil {
t.Fatalf("Filtered page 1 search failed: %v", err)
}
@@ -1500,7 +1504,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
t.Error("Expected results on filtered page 1")
}
gistIDs2, _, _, err := indexer.Search("", metadata, 1, 2)
gistIDs2, _, _, err := indexer.Search(metadata, 1, 2)
if err != nil {
t.Fatalf("Filtered page 2 search failed: %v", err)
}
@@ -1518,7 +1522,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
t.Run("LastPageVerification", func(t *testing.T) {
// With 334 results and page size 10, page 34 should have 4 results
// Let's just verify the last page has some results
gistIDs34, total, _, err := indexer.Search("", SearchGistMetadata{}, 1, 34)
gistIDs34, total, _, err := indexer.Search(SearchGistMetadata{}, 1, 34)
if err != nil {
t.Fatalf("Last page search failed: %v", err)
}
@@ -1530,7 +1534,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
}
// Page 35 should be empty
gistIDs35, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 35)
gistIDs35, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 35)
if err != nil {
t.Fatalf("Beyond last page search failed: %v", err)
}
@@ -1541,15 +1545,15 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
// Test 9: Multiple pages have different results
t.Run("MultiplePagesDifferent", func(t *testing.T) {
gistIDs1, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 1)
gistIDs1, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 1)
if err != nil {
t.Fatalf("Page 1 search failed: %v", err)
}
gistIDs10, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 10)
gistIDs10, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 10)
if err != nil {
t.Fatalf("Page 10 search failed: %v", err)
}
gistIDs20, _, _, err := indexer.Search("", SearchGistMetadata{}, 1, 20)
gistIDs20, _, _, err := indexer.Search(SearchGistMetadata{}, 1, 20)
if err != nil {
t.Fatalf("Page 20 search failed: %v", err)
}
@@ -1568,7 +1572,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
// Test 10: Pagination with different users (visibility filtering)
t.Run("PaginationWithVisibility", func(t *testing.T) {
// User 2 (bob) sees 667 gists (334 public alice + 333 own private)
gistIDs1Bob, totalBob, _, err := indexer.Search("", SearchGistMetadata{}, 2, 1)
gistIDs1Bob, totalBob, _, err := indexer.Search(SearchGistMetadata{}, 2, 1)
if err != nil {
t.Fatalf("Bob page 1 search failed: %v", err)
}
@@ -1580,7 +1584,7 @@ func testIndexerPagination(t *testing.T, indexer Indexer) {
}
// User 1 (alice) sees 334 gists
_, totalAlice, _, err := indexer.Search("", SearchGistMetadata{}, 1, 1)
_, totalAlice, _, err := indexer.Search(SearchGistMetadata{}, 1, 1)
if err != nil {
t.Fatalf("Alice page 1 search failed: %v", err)
}

View File

@@ -9,6 +9,7 @@ import (
"github.com/meilisearch/meilisearch-go"
"github.com/rs/zerolog/log"
"github.com/thomiceli/opengist/internal/config"
)
type MeiliIndexer struct {
@@ -63,9 +64,9 @@ func (i *MeiliIndexer) open() (meilisearch.IndexManager, error) {
}
_, _ = i.client.Index(i.indexName).UpdateSettings(&meilisearch.Settings{
FilterableAttributes: []string{"GistID", "UserID", "Visibility", "Username", "Title", "Filenames", "Extensions", "Languages", "Topics"},
FilterableAttributes: []string{"GistID", "UserID", "Visibility", "Username", "Title", "Description", "Filenames", "Extensions", "Languages", "Topics"},
DisplayedAttributes: []string{"GistID"},
SearchableAttributes: []string{"Content", "Username", "Title", "Filenames", "Extensions", "Languages", "Topics"},
SearchableAttributes: []string{"Content", "Username", "Title", "Description", "Filenames", "Extensions", "Languages", "Topics"},
RankingRules: []string{"words"},
})
@@ -109,7 +110,7 @@ func (i *MeiliIndexer) Remove(gistID uint) error {
return err
}
func (i *MeiliIndexer) Search(queryStr string, queryMetadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
func (i *MeiliIndexer) Search(queryMetadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
searchRequest := &meilisearch.SearchRequest{
Offset: int64((page - 1) * 10),
Limit: 11,
@@ -128,6 +129,7 @@ func (i *MeiliIndexer) Search(queryStr string, queryMetadata SearchGistMetadata,
}
addFilter("Username", queryMetadata.Username)
addFilter("Title", queryMetadata.Title)
addFilter("Description", queryMetadata.Description)
addFilter("Filenames", queryMetadata.Filename)
addFilter("Extensions", queryMetadata.Extension)
addFilter("Languages", queryMetadata.Language)
@@ -137,7 +139,29 @@ func (i *MeiliIndexer) Search(queryStr string, queryMetadata SearchGistMetadata,
searchRequest.Filter = strings.Join(filters, " AND ")
}
response, err := (*atomicIndexer.Load()).(*MeiliIndexer).index.Search(queryStr, searchRequest)
// build query string from provided metadata. Prefer `All`, then `Default`, fall back to `Content`.
query := queryMetadata.All
if query == "" && queryMetadata.Default != "" {
query = queryMetadata.Default
var fields []string
for _, f := range strings.Split(config.C.SearchDefault, ",") {
f = strings.TrimSpace(f)
if f == "all" {
fields = AllSearchFields
break
}
if indexField, ok := SearchFieldMap[f]; ok {
fields = append(fields, indexField)
}
}
if len(fields) > 0 {
searchRequest.AttributesToSearchOn = fields
}
} else if query == "" {
query = queryMetadata.Content
}
response, err := (*atomicIndexer.Load()).(*MeiliIndexer).index.Search(query, searchRequest)
if err != nil {
log.Error().Err(err).Msg("Failed to search Meilisearch index")
return nil, 0, nil, err

View File

@@ -164,6 +164,18 @@ func AllGists(ctx *context.Context) error {
return ctx.Html("all.html")
}
// Search handles the search page for gists.
//
// It takes a query parameter "q" which is a search query in the format:
// "user:username title:title description:description filename:filename language:language topic:topic"
//
// It also takes a page parameter "page" which is the page number to display.
//
// It returns an error if the search query is invalid or if the page number is invalid.
//
// It returns the search results as a list of rendered gists, along with the total number of results, the languages found, and the search query.
//
// The search results are paginated, with 10 results per page.
func Search(ctx *context.Context) error {
var err error
@@ -171,7 +183,7 @@ func Search(ctx *context.Context) error {
Query: ctx.QueryParam("q"),
}
content, meta := handlers.ParseSearchQueryStr(ctx.QueryParam("q"))
metadata := handlers.ParseSearchQueryStr(ctx.QueryParam("q"))
pageInt := handlers.GetPage(ctx)
var currentUserId uint
@@ -182,14 +194,18 @@ func Search(ctx *context.Context) error {
currentUserId = 0
}
gistsIds, nbHits, langs, err := index.SearchGists(content, index.SearchGistMetadata{
Username: meta["user"],
Title: meta["title"],
Filename: meta["filename"],
Extension: meta["extension"],
Language: meta["language"],
Topic: meta["topic"],
All: meta["all"],
// Search gists in the index and fetch the gists IDs from the database
gistsIds, nbHits, langs, err := index.SearchGists(index.SearchGistMetadata{
Username: metadata["user"],
Title: metadata["title"],
Description: metadata["description"],
Filename: metadata["filename"],
Extension: metadata["extension"],
Language: metadata["language"],
Topic: metadata["topic"],
Content: metadata["content"],
All: metadata["all"],
Default: metadata["default"],
}, currentUserId, pageInt)
if err != nil {
return ctx.ErrorRes(500, "Error searching gists", err)

View File

@@ -8,7 +8,7 @@ import (
"strings"
"github.com/gorilla/schema"
"github.com/rs/zerolog/log"
"github.com/thomiceli/opengist/internal/web/context"
)
@@ -119,10 +119,16 @@ func Paginate[T any](ctx *context.Context, data []*T, pageInt int, perPage int,
return nil
}
func ParseSearchQueryStr(query string) (string, map[string]string) {
// ParseSearchQueryStr parses a search query string and returns a map of metadata.
// The query string is split into words and each word is checked if it contains a colon (:).
// If a word contains a colon, it is split into a key-value pair and added to the metadata map.
// If a word does not contain a colon, it is added to an "all" key in the metadata map.
// The "all" key is used to search all fields in the index.
// The function returns the metadata map.
func ParseSearchQueryStr(query string) map[string]string {
words := strings.Fields(query)
metadata := make(map[string]string)
var contentBuilder strings.Builder
var allFieldsBuilder strings.Builder
for _, word := range words {
if strings.Contains(word, ":") {
@@ -133,10 +139,18 @@ func ParseSearchQueryStr(query string) (string, map[string]string) {
metadata[key] = value
}
} else {
contentBuilder.WriteString(word + " ")
// Add to content search by default
allFieldsBuilder.WriteString(word + " ")
}
}
content := strings.TrimSpace(contentBuilder.String())
return content, metadata
// Set the default search field
allContent := strings.TrimSpace(allFieldsBuilder.String())
if allContent != "" {
metadata["default"] = allContent
}
log.Debug().Msgf("Metadata: %v", metadata)
return metadata
}

View File

@@ -147,7 +147,10 @@ func (s *Server) setFuncMap() {
return dict, nil
},
"addMetadataToSearchQuery": func(input, key, value string) string {
content, metadata := handlers.ParseSearchQueryStr(input)
metadata := handlers.ParseSearchQueryStr(input)
// extract free-text content (stored under "all") and remove it from metadata
content := metadata["all"]
delete(metadata, "all")
metadata[key] = value

View File

@@ -116,10 +116,12 @@
<div class="p-4 text-xs space-y-1">
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">user:thomas</code> {{ .locale.Tr "gist.search.help.user" }}</p>
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">title:mygist</code> {{ .locale.Tr "gist.search.help.title" }}</p>
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">description:sync</code> {{ .locale.Tr "gist.search.help.description" }}</p>
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">filename:myfile.txt</code> {{ .locale.Tr "gist.search.help.filename" }}</p>
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">extension:yml</code> {{ .locale.Tr "gist.search.help.extension" }}</p>
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">language:go</code> {{ .locale.Tr "gist.search.help.language" }}</p>
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">topic:homelab</code> {{ .locale.Tr "gist.search.help.topic" }}</p>
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">all:systemctl</code> {{ .locale.Tr "gist.search.help.all" }}</p>
</div>
</div>
</div>