Compare commits

..

1 Commits

Author SHA1 Message Date
Anonymous
69b9b215d6 Translated using Weblate (Ukrainian)
Currently translated at 69.4% (243 of 350 strings)

Translation: Opengist/Opengist
Translate-URL: http://tr.opengist.io/projects/_/opengist/uk/
2026-03-09 23:52:08 +00:00
23 changed files with 2242 additions and 1350 deletions

View File

@@ -83,18 +83,6 @@ jobs:
--health-interval 10s
--health-timeout 5s
--health-retries 5
meilisearch:
image: getmeili/meilisearch:latest
ports:
- 47700:7700
env:
MEILI_NO_ANALYTICS: true
MEILI_ENV: development
options: >-
--health-cmd "curl -sf http://localhost:7700/health"
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- name: Checkout
uses: actions/checkout@v6
@@ -106,8 +94,6 @@ jobs:
- name: Run tests
run: make test TEST_DB_TYPE=${{ matrix.database }}
env:
OG_TEST_MEILI_HOST: http://localhost:47700
test:
name: Test

View File

@@ -42,7 +42,7 @@ jobs:
- name: Docker meta
id: meta
uses: docker/metadata-action@v6
uses: docker/metadata-action@v5
with:
images: |
ghcr.io/thomiceli/opengist
@@ -54,26 +54,26 @@ jobs:
type=semver,pattern={{version}}
- name: Set up QEMU
uses: docker/setup-qemu-action@v4
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Login to GitHub Container Registry
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to Docker Hub
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and push
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64,linux/arm64

View File

@@ -32,10 +32,6 @@ index.meili.host:
# Set the API key for the Meiliseach server
index.meili.api-key:
# Set the default search fields. Can contain multiple fields (e.g., `content,username`).
# Fields: content,user,title,description,filename,extension,language,topic. Default: content
search.default: content
# Default branch name used by Opengist when initializing Git repositories.
# If not set, uses the Git default branch name. See https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup#_new_default_branch
git.default-branch:

View File

@@ -15,7 +15,6 @@ aside: false
| index | OG_INDEX | `bleve` | Define the code indexer (either `bleve`, `meilisearch`, or empty for no index). |
| index.meili.host | OG_MEILI_HOST | none | Set the host for the Meiliseach server. |
| index.meili.api-key | OG_MEILI_API_KEY | none | Set the API key for the Meiliseach server. |
| search.default | OG_SEARCH_DEFAULT | `content` | Set the default search fields. Can contain multiple fields (e.g., `content,username`). Fields: `content,user,title,description,filename,extension,language,topic`. |
| git.default-branch | OG_GIT_DEFAULT_BRANCH | none | Default branch name used by Opengist when initializing Git repositories. If not set, uses the Git default branch name. More info [here](https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup#_new_default_branch) |
| sqlite.journal-mode | OG_SQLITE_JOURNAL_MODE | `WAL` | Set the journal mode for SQLite. More info [here](https://www.sqlite.org/pragma.html#pragma_journal_mode) |
| http.host | OG_HTTP_HOST | `0.0.0.0` | The host on which the HTTP server should bind. Use an IP address for network binding. Use a path for Unix socket binding (e.g. /run/opengist.sock) |

View File

@@ -43,7 +43,6 @@ type config struct {
BleveDirname string `yaml:"index.dirname" env:"OG_INDEX_DIRNAME"` // deprecated
MeiliHost string `yaml:"index.meili.host" env:"OG_MEILI_HOST"`
MeiliAPIKey string `yaml:"index.meili.api-key" env:"OG_MEILI_API_KEY"`
SearchDefault string `yaml:"search.default" env:"OG_SEARCH_DEFAULT"`
GitDefaultBranch string `yaml:"git.default-branch" env:"OG_GIT_DEFAULT_BRANCH"`
@@ -112,7 +111,6 @@ func configWithDefaults() (*config, error) {
c.OpengistHome = ""
c.DBUri = "opengist.db"
c.Index = "bleve"
c.SearchDefault = "content"
c.SqliteJournalMode = "WAL"

View File

@@ -821,7 +821,6 @@ func (gist *Gist) ToIndexedGist() (*index.Gist, error) {
UserID: gist.UserID,
Visibility: gist.Private.Uint(),
Username: gist.User.Username,
Description: gist.Description,
Title: gist.Title,
Content: wholeContent,
Filenames: fileNames,

View File

@@ -88,12 +88,10 @@ gist.search.found: gists found
gist.search.no-results: No gists found
gist.search.help.user: gists created by user
gist.search.help.title: gists with given title
gist.search.help.description: gists with given description
gist.search.help.filename: gists having files with given name
gist.search.help.extension: gists having files with given extension
gist.search.help.language: gists having files with given language
gist.search.help.topic: gists with given topic
gist.search.help.all: search all fields
gist.search.placeholder.title: Title
gist.search.placeholder.visibility: Visibility
gist.search.placeholder.public: Public

View File

@@ -5,18 +5,15 @@ import (
"fmt"
"os"
"strconv"
"strings"
"github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
"github.com/blevesearch/bleve/v2/analysis/token/camelcase"
"github.com/blevesearch/bleve/v2/analysis/token/length"
"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
"github.com/blevesearch/bleve/v2/analysis/token/unicodenorm"
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
"github.com/blevesearch/bleve/v2/search/query"
"github.com/rs/zerolog/log"
"github.com/thomiceli/opengist/internal/config"
)
type BleveIndexer struct {
@@ -57,9 +54,14 @@ func (i *BleveIndexer) open() (bleve.Index, error) {
return nil, err
}
docMapping := bleve.NewDocumentMapping()
docMapping.AddFieldMappingsAt("GistID", bleve.NewNumericFieldMapping())
docMapping.AddFieldMappingsAt("UserID", bleve.NewNumericFieldMapping())
docMapping.AddFieldMappingsAt("Visibility", bleve.NewNumericFieldMapping())
docMapping.AddFieldMappingsAt("Content", bleve.NewTextFieldMapping())
mapping := bleve.NewIndexMapping()
// Token filters
if err = mapping.AddCustomTokenFilter("unicodeNormalize", map[string]any{
"type": unicodenorm.Name,
"form": unicodenorm.NFC,
@@ -67,74 +69,16 @@ func (i *BleveIndexer) open() (bleve.Index, error) {
return nil, err
}
if err = mapping.AddCustomTokenFilter("lengthMin2", map[string]interface{}{
"type": length.Name,
"min": 2.0,
}); err != nil {
return nil, err
}
// Analyzer: split mode (camelCase splitting for partial search)
// "CPUCard" -> ["cpu", "card"]
if err = mapping.AddCustomAnalyzer("codeSplit", map[string]interface{}{
if err = mapping.AddCustomAnalyzer("gistAnalyser", map[string]interface{}{
"type": custom.Name,
"char_filters": []string{},
"tokenizer": unicode.Name,
"token_filters": []string{"unicodeNormalize", camelcase.Name, lowercase.Name, "lengthMin2"},
"token_filters": []string{"unicodeNormalize", camelcase.Name, lowercase.Name},
}); err != nil {
return nil, err
}
// Analyzer: exact mode (no camelCase splitting for full-word search)
// "CPUCard" -> ["cpucard"]
if err = mapping.AddCustomAnalyzer("codeExact", map[string]interface{}{
"type": custom.Name,
"char_filters": []string{},
"tokenizer": unicode.Name,
"token_filters": []string{"unicodeNormalize", lowercase.Name},
}); err != nil {
return nil, err
}
// Analyzer: keyword with lowercase (for Languages - single token, no splitting)
if err = mapping.AddCustomAnalyzer("lowercaseKeyword", map[string]interface{}{
"type": custom.Name,
"char_filters": []string{},
"tokenizer": "single",
"token_filters": []string{lowercase.Name},
}); err != nil {
return nil, err
}
// Document mapping
docMapping := bleve.NewDocumentMapping()
docMapping.AddFieldMappingsAt("GistID", bleve.NewNumericFieldMapping())
docMapping.AddFieldMappingsAt("UserID", bleve.NewNumericFieldMapping())
docMapping.AddFieldMappingsAt("Visibility", bleve.NewNumericFieldMapping())
// Content: dual indexing (exact + split)
// "Content" uses the property name so Bleve resolves its analyzer correctly
contentExact := bleve.NewTextFieldMapping()
contentExact.Name = "Content"
contentExact.Analyzer = "codeExact"
contentExact.Store = false
contentExact.IncludeTermVectors = true
contentSplit := bleve.NewTextFieldMapping()
contentSplit.Name = "ContentSplit"
contentSplit.Analyzer = "codeSplit"
contentSplit.Store = false
contentSplit.IncludeTermVectors = true
docMapping.AddFieldMappingsAt("Content", contentExact, contentSplit)
// Languages: keyword analyzer (preserves as single token)
languageFieldMapping := bleve.NewTextFieldMapping()
languageFieldMapping.Analyzer = "lowercaseKeyword"
docMapping.AddFieldMappingsAt("Languages", languageFieldMapping)
// All other text fields use codeSplit as default
docMapping.DefaultAnalyzer = "codeSplit"
docMapping.DefaultAnalyzer = "gistAnalyser"
mapping.DefaultMapping = docMapping
return bleve.New(i.path, mapping)
@@ -172,111 +116,18 @@ func (i *BleveIndexer) Remove(gistID uint) error {
return (*atomicIndexer.Load()).(*BleveIndexer).index.Delete(strconv.Itoa(int(gistID)))
}
// Search returns a list of Gist IDs that match the given search metadata.
// The method returns an error if any.
//
// The queryMetadata parameter is used to filter the search results.
// For example, passing a non-empty Username will search for gists whose
// username matches the given string.
//
// If the "All" field in queryMetadata is non-empty, the method will
// search across all metadata fields with OR logic. Otherwise, the method
// will add each metadata field with AND logic.
//
// The page parameter is used to paginate the search results.
// The method returns the total number of search results in the second return
// value.
//
// The third return value is a map of language counts for the search results.
// The language counts are computed by asking Bleve to return the top 10
// facets for the "Languages" field.
func (i *BleveIndexer) Search(metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
func (i *BleveIndexer) Search(queryStr string, queryMetadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
var err error
var indexerQuery query.Query = bleve.NewMatchAllQuery()
// Query factory
factoryQuery := func(field, value string) query.Query {
query := bleve.NewMatchPhraseQuery(value)
query.SetField(field)
return query
}
// Exact search
addQuery := func(field, value string) {
if value != "" && value != "." {
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, factoryQuery(field, value))
}
}
// Query factory for text fields: exact match boosted + match query + prefix
factoryTextQuery := func(field, value string) query.Query {
exact := bleve.NewMatchPhraseQuery(value)
exact.SetField(field)
exact.SetBoost(2.0)
fuzzy := bleve.NewMatchQuery(value)
fuzzy.SetField(field)
fuzzy.SetFuzziness(1)
fuzzy.SetOperator(query.MatchQueryOperatorAnd)
queries := []query.Query{exact, fuzzy}
if len([]rune(value)) >= 2 {
prefix := bleve.NewPrefixQuery(strings.ToLower(value))
prefix.SetField(field)
prefix.SetBoost(1.5)
queries = append(queries, prefix)
}
if len([]rune(value)) >= 4 {
wildcard := bleve.NewWildcardQuery("*" + strings.ToLower(value) + "*")
wildcard.SetField(field)
wildcard.SetBoost(0.5)
queries = append(queries, wildcard)
}
return bleve.NewDisjunctionQuery(queries...)
}
// Query factory for Content: searches both exact (Content) and split (ContentSplit) fields
factoryContentQuery := func(value string) query.Query {
// Exact field (no camelCase split): matches "cpucard"
exactMatch := bleve.NewMatchQuery(value)
exactMatch.SetField("Content")
exactMatch.SetOperator(query.MatchQueryOperatorAnd)
exactMatch.SetBoost(2.0)
// Split field (camelCase split): matches "cpu", "card"
splitMatch := bleve.NewMatchQuery(value)
splitMatch.SetField("ContentSplit")
splitMatch.SetFuzziness(1)
splitMatch.SetOperator(query.MatchQueryOperatorAnd)
splitMatch.SetBoost(1.0)
queries := []query.Query{exactMatch, splitMatch}
if len([]rune(value)) >= 2 {
prefix := bleve.NewPrefixQuery(strings.ToLower(value))
prefix.SetField("Content")
prefix.SetBoost(1.5)
queries = append(queries, prefix)
}
if len([]rune(value)) >= 4 {
wildcard := bleve.NewWildcardQuery("*" + strings.ToLower(value) + "*")
wildcard.SetField("Content")
wildcard.SetBoost(0.5)
queries = append(queries, wildcard)
}
return bleve.NewDisjunctionQuery(queries...)
}
// Text field search
addTextQuery := func(field, value string) {
if value != "" && value != "." {
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, factoryTextQuery(field, value))
}
var indexerQuery query.Query
if queryStr != "" {
// Use match query with fuzzy matching for more flexible content search
contentQuery := bleve.NewMatchQuery(queryStr)
contentQuery.SetField("Content")
contentQuery.SetFuzziness(2)
indexerQuery = contentQuery
} else {
contentQuery := bleve.NewMatchAllQuery()
indexerQuery = contentQuery
}
// Visibility filtering: show public gists (Visibility=0) OR user's own gists
@@ -292,62 +143,48 @@ func (i *BleveIndexer) Search(metadata SearchGistMetadata, userId uint, page int
accessQuery := bleve.NewDisjunctionQuery(publicQuery, userIdQuery)
indexerQuery = bleve.NewConjunctionQuery(accessQuery, indexerQuery)
buildFieldQuery := func(field, value string) query.Query {
switch field {
case "Content":
return factoryContentQuery(value)
case "Title", "Description", "Filenames":
return factoryTextQuery(field, value)
case "Extensions":
return factoryQuery(field, "."+value)
default: // Username, Languages, Topics
return factoryQuery(field, value)
}
// Handle "All" field - search across all metadata fields with OR logic
if queryMetadata.All != "" {
allQueries := make([]query.Query, 0)
// Create match phrase queries for each field
fields := []struct {
field string
value string
}{
{"Username", queryMetadata.All},
{"Title", queryMetadata.All},
{"Extensions", "." + queryMetadata.All},
{"Filenames", queryMetadata.All},
{"Languages", queryMetadata.All},
{"Topics", queryMetadata.All},
}
// Handle "All" field - search across all metadata fields with OR logic
if metadata.All != "" {
allQueries := make([]query.Query, 0, len(AllSearchFields))
for _, field := range AllSearchFields {
allQueries = append(allQueries, buildFieldQuery(field, metadata.All))
for _, f := range fields {
q := bleve.NewMatchPhraseQuery(f.value)
q.FieldVal = f.field
allQueries = append(allQueries, q)
}
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, bleve.NewDisjunctionQuery(allQueries...))
// Combine all field queries with OR (disjunction)
allDisjunction := bleve.NewDisjunctionQuery(allQueries...)
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, allDisjunction)
} else {
// Original behavior: add each metadata field with AND logic
addQuery("Username", metadata.Username)
addTextQuery("Title", metadata.Title)
addTextQuery("Description", metadata.Description)
addQuery("Extensions", "."+metadata.Extension)
addTextQuery("Filenames", metadata.Filename)
addQuery("Languages", metadata.Language)
addQuery("Topics", metadata.Topic)
if metadata.Content != "" {
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, factoryContentQuery(metadata.Content))
addQuery := func(field, value string) {
if value != "" && value != "." {
q := bleve.NewMatchPhraseQuery(value)
q.FieldVal = field
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, q)
}
}
// Handle default search fields from config with OR logic
if metadata.Default != "" {
var fields []string
for _, f := range strings.Split(config.C.SearchDefault, ",") {
f = strings.TrimSpace(f)
if f == "all" {
fields = AllSearchFields
break
}
if indexField, ok := SearchFieldMap[f]; ok {
fields = append(fields, indexField)
}
}
if len(fields) == 1 {
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, buildFieldQuery(fields[0], metadata.Default))
} else if len(fields) > 1 {
defaultQueries := make([]query.Query, 0, len(fields))
for _, field := range fields {
defaultQueries = append(defaultQueries, buildFieldQuery(field, metadata.Default))
}
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, bleve.NewDisjunctionQuery(defaultQueries...))
}
}
addQuery("Username", queryMetadata.Username)
addQuery("Title", queryMetadata.Title)
addQuery("Extensions", "."+queryMetadata.Extension)
addQuery("Filenames", queryMetadata.Filename)
addQuery("Languages", queryMetadata.Language)
addQuery("Topics", queryMetadata.Topic)
}
languageFacet := bleve.NewFacetRequest("Languages", 10)
@@ -360,8 +197,6 @@ func (i *BleveIndexer) Search(metadata SearchGistMetadata, userId uint, page int
s.Fields = []string{"GistID"}
s.IncludeLocations = false
log.Debug().Interface("searchRequest", s).Msg("Bleve search request")
results, err := (*atomicIndexer.Load()).(*BleveIndexer).index.Search(s)
if err != nil {
return nil, 0, nil, err

View File

@@ -4,31 +4,33 @@ import (
"os"
"path/filepath"
"testing"
"github.com/rs/zerolog"
"github.com/stretchr/testify/require"
)
// setupBleveIndexer creates a new BleveIndexer for testing
func setupBleveIndexer(t *testing.T) (Indexer, func()) {
zerolog.SetGlobalLevel(zerolog.Disabled)
func setupBleveIndexer(t *testing.T) (*BleveIndexer, func()) {
t.Helper()
// Create a temporary directory for the test index
tmpDir, err := os.MkdirTemp("", "bleve-test-*")
require.NoError(t, err)
if err != nil {
t.Fatalf("Failed to create temp directory: %v", err)
}
indexPath := filepath.Join(tmpDir, "test.index")
indexer := NewBleveIndexer(indexPath)
// Initialize the indexer
err = indexer.Init()
if err != nil {
os.RemoveAll(tmpDir)
t.Fatalf("Failed to initialize BleveIndexer: %v", err)
}
// Store in the global atomicIndexer since Add/Remove use it
var idx Indexer = indexer
atomicIndexer.Store(&idx)
// Return cleanup function
cleanup := func() {
atomicIndexer.Store(nil)
indexer.Close()
@@ -38,50 +40,123 @@ func setupBleveIndexer(t *testing.T) (Indexer, func()) {
return indexer, cleanup
}
func TestBleveAddAndSearch(t *testing.T) { testAddAndSearch(t, setupBleveIndexer) }
func TestBleveAccessControl(t *testing.T) { testAccessControl(t, setupBleveIndexer) }
func TestBleveMetadataFilters(t *testing.T) { testMetadataFilters(t, setupBleveIndexer) }
func TestBleveAllFieldSearch(t *testing.T) { testAllFieldSearch(t, setupBleveIndexer) }
func TestBleveFuzzySearch(t *testing.T) { testFuzzySearch(t, setupBleveIndexer) }
func TestBleveContentSearch(t *testing.T) { testContentSearch(t, setupBleveIndexer) }
func TestBlevePagination(t *testing.T) { testPagination(t, setupBleveIndexer) }
func TestBleveLanguageFacets(t *testing.T) { testLanguageFacets(t, setupBleveIndexer) }
func TestBleveWildcardSearch(t *testing.T) { testWildcardSearch(t, setupBleveIndexer) }
func TestBleveMetadataOnlySearch(t *testing.T) { testMetadataOnlySearch(t, setupBleveIndexer) }
func TestBleveTitleFuzzySearch(t *testing.T) { testTitleFuzzySearch(t, setupBleveIndexer) }
func TestBleveMultiLanguageFacets(t *testing.T) { testMultiLanguageFacets(t, setupBleveIndexer) }
func TestBleveIndexerAddGist(t *testing.T) {
indexer, cleanup := setupBleveIndexer(t)
defer cleanup()
func TestBlevePersistence(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "bleve-persist-test-*")
require.NoError(t, err)
testIndexerAddGist(t, indexer)
}
func TestBleveIndexerAllFieldSearch(t *testing.T) {
indexer, cleanup := setupBleveIndexer(t)
defer cleanup()
testIndexerAllFieldSearch(t, indexer)
}
func TestBleveIndexerFuzzySearch(t *testing.T) {
indexer, cleanup := setupBleveIndexer(t)
defer cleanup()
testIndexerFuzzySearch(t, indexer)
}
func TestBleveIndexerSearchBasic(t *testing.T) {
indexer, cleanup := setupBleveIndexer(t)
defer cleanup()
testIndexerSearchBasic(t, indexer)
}
func TestBleveIndexerPagination(t *testing.T) {
indexer, cleanup := setupBleveIndexer(t)
defer cleanup()
testIndexerPagination(t, indexer)
}
// TestBleveIndexerInitAndClose tests Bleve-specific initialization and closing
func TestBleveIndexerInitAndClose(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "bleve-init-test-*")
if err != nil {
t.Fatalf("Failed to create temp directory: %v", err)
}
defer os.RemoveAll(tmpDir)
indexPath := filepath.Join(tmpDir, "test.index")
indexer := NewBleveIndexer(indexPath)
// Create and populate index
indexer1 := NewBleveIndexer(indexPath)
require.NoError(t, indexer1.Init())
// Test initialization
err = indexer.Init()
if err != nil {
t.Fatalf("Failed to initialize BleveIndexer: %v", err)
}
var idx Indexer = indexer1
atomicIndexer.Store(&idx)
if indexer.index == nil {
t.Fatal("Expected index to be initialized, got nil")
}
g := newGist(1, 1, 0, "persistent data survives restart")
require.NoError(t, indexer1.Add(g))
// Test closing
indexer.Close()
indexer1.Close()
atomicIndexer.Store(nil)
// Reopen at same path
// Test reopening the same index
indexer2 := NewBleveIndexer(indexPath)
require.NoError(t, indexer2.Init())
err = indexer2.Init()
if err != nil {
t.Fatalf("Failed to reopen BleveIndexer: %v", err)
}
defer indexer2.Close()
idx = indexer2
atomicIndexer.Store(&idx)
defer atomicIndexer.Store(nil)
ids, total, _, err := indexer2.Search(SearchGistMetadata{Content: "persistent"}, 1, 1)
require.NoError(t, err)
require.Equal(t, uint64(1), total, "data should survive close+reopen")
require.Equal(t, uint(1), ids[0])
if indexer2.index == nil {
t.Fatal("Expected reopened index to be initialized, got nil")
}
}
// TestBleveIndexerUnicodeSearch tests that Unicode content can be indexed and searched
func TestBleveIndexerUnicodeSearch(t *testing.T) {
indexer, cleanup := setupBleveIndexer(t)
defer cleanup()
// Add a gist with Unicode content
gist := &Gist{
GistID: 100,
UserID: 100,
Visibility: 0,
Username: "testuser",
Title: "Unicode Test",
Content: "Hello world with unicode characters: café résumé naïve",
Filenames: []string{"test.txt"},
Extensions: []string{".txt"},
Languages: []string{"Text"},
Topics: []string{"unicode"},
CreatedAt: 1234567890,
UpdatedAt: 1234567890,
}
err := indexer.Add(gist)
if err != nil {
t.Fatalf("Failed to add gist: %v", err)
}
// Search for unicode content
gistIDs, total, _, err := indexer.Search("café", SearchGistMetadata{}, 100, 1)
if err != nil {
t.Fatalf("Search failed: %v", err)
}
if total == 0 {
t.Skip("Unicode search may require specific index configuration")
return
}
found := false
for _, id := range gistIDs {
if id == 100 {
found = true
break
}
}
if !found {
t.Log("Unicode gist not found in search results, but other results were returned")
}
}

View File

@@ -1,24 +1,10 @@
package index
var AllSearchFields = []string{"Username", "Title", "Description", "Filenames", "Extensions", "Languages", "Topics", "Content"}
var SearchFieldMap = map[string]string{
"user": "Username",
"title": "Title",
"description": "Description",
"filename": "Filenames",
"extension": "Extensions",
"language": "Languages",
"topic": "Topics",
"content": "Content",
}
type Gist struct {
GistID uint
UserID uint
Visibility uint
Username string
Description string
Title string
Content string
Filenames []string
@@ -32,12 +18,9 @@ type Gist struct {
type SearchGistMetadata struct {
Username string
Title string
Description string
Content string
Filename string
Extension string
Language string
Topic string
All string
Default string
}

View File

@@ -17,7 +17,7 @@ type Indexer interface {
Reset() error
Add(gist *Gist) error
Remove(gistID uint) error
Search(metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error)
Search(query string, metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error)
}
type IndexerType string
@@ -125,11 +125,7 @@ func RemoveFromIndex(gistID uint) error {
return (*idx).Remove(gistID)
}
// SearchGists returns a list of Gist IDs that match the given search metadata.
// If the indexer is not enabled, it returns nil, 0, nil, nil.
// If the indexer is not initialized, it returns nil, 0, nil, fmt.Errorf("indexer is not initialized").
// The function returns an error if any.
func SearchGists(metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
func SearchGists(query string, metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
if !IndexEnabled() {
return nil, 0, nil, nil
}
@@ -139,7 +135,7 @@ func SearchGists(metadata SearchGistMetadata, userId uint, page int) ([]uint, ui
return nil, 0, nil, fmt.Errorf("indexer is not initialized")
}
return (*idx).Search(metadata, userId, page)
return (*idx).Search(query, metadata, userId, page)
}
func DepreactionIndexDirname() {

File diff suppressed because it is too large Load Diff

View File

@@ -6,11 +6,9 @@ import (
"fmt"
"strconv"
"strings"
"unicode"
"github.com/meilisearch/meilisearch-go"
"github.com/rs/zerolog/log"
"github.com/thomiceli/opengist/internal/config"
)
type MeiliIndexer struct {
@@ -52,7 +50,10 @@ func (i *MeiliIndexer) open() (meilisearch.IndexManager, error) {
i.client = meilisearch.New(i.host, meilisearch.WithAPIKey(i.apikey))
indexResult, err := i.client.GetIndex(i.indexName)
if indexResult == nil || err != nil {
if indexResult != nil && err == nil {
return indexResult.IndexManager, nil
}
_, err = i.client.CreateIndex(&meilisearch.IndexConfig{
Uid: i.indexName,
PrimaryKey: "GistID",
@@ -60,17 +61,12 @@ func (i *MeiliIndexer) open() (meilisearch.IndexManager, error) {
if err != nil {
return nil, err
}
}
_, _ = i.client.Index(i.indexName).UpdateSettings(&meilisearch.Settings{
FilterableAttributes: []string{"GistID", "UserID", "Visibility", "Username", "Extensions", "Languages", "Topics"},
SearchableAttributes: []string{"Content", "ContentSplit", "Username", "Title", "Description", "Filenames", "Extensions", "Languages", "Topics"},
RankingRules: []string{"words", "typo", "proximity", "attribute", "sort", "exactness"},
TypoTolerance: &meilisearch.TypoTolerance{
Enabled: true,
DisableOnNumbers: true,
MinWordSizeForTypos: meilisearch.MinWordSizeForTypos{OneTypo: 4, TwoTypos: 10},
},
FilterableAttributes: []string{"GistID", "UserID", "Visibility", "Username", "Title", "Filenames", "Extensions", "Languages", "Topics"},
DisplayedAttributes: []string{"GistID"},
SearchableAttributes: []string{"Content", "Username", "Title", "Filenames", "Extensions", "Languages", "Topics"},
RankingRules: []string{"words"},
})
return i.client.Index(i.indexName), nil
@@ -99,21 +95,12 @@ func (i *MeiliIndexer) Close() {
i.client = nil
}
type meiliGist struct {
Gist
ContentSplit string
}
func (i *MeiliIndexer) Add(gist *Gist) error {
if gist == nil {
return errors.New("failed to add nil gist to index")
}
doc := &meiliGist{
Gist: *gist,
ContentSplit: splitCamelCase(gist.Content),
}
primaryKey := "GistID"
_, err := (*atomicIndexer.Load()).(*MeiliIndexer).index.AddDocuments(doc, &meilisearch.DocumentOptions{PrimaryKey: &primaryKey})
_, err := (*atomicIndexer.Load()).(*MeiliIndexer).index.AddDocuments(gist, &meilisearch.DocumentOptions{PrimaryKey: &primaryKey})
return err
}
@@ -122,14 +109,13 @@ func (i *MeiliIndexer) Remove(gistID uint) error {
return err
}
func (i *MeiliIndexer) Search(queryMetadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
func (i *MeiliIndexer) Search(queryStr string, queryMetadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
searchRequest := &meilisearch.SearchRequest{
Offset: int64((page - 1) * 10),
Limit: 11,
AttributesToRetrieve: []string{"GistID", "Languages"},
Facets: []string{"Languages"},
AttributesToSearchOn: []string{"Content", "ContentSplit"},
MatchingStrategy: meilisearch.All,
AttributesToSearchOn: []string{"Content"},
}
var filters []string
@@ -140,83 +126,23 @@ func (i *MeiliIndexer) Search(queryMetadata SearchGistMetadata, userId uint, pag
filters = append(filters, fmt.Sprintf("%s = \"%s\"", field, escapeFilterValue(value)))
}
}
var query string
if queryMetadata.All != "" {
query = queryMetadata.All
searchRequest.AttributesToSearchOn = append(AllSearchFields, "ContentSplit")
} else {
// Exact-match fields stay as filters
addFilter("Username", queryMetadata.Username)
if queryMetadata.Extension != "" {
ext := queryMetadata.Extension
if !strings.HasPrefix(ext, ".") {
ext = "." + ext
}
addFilter("Extensions", ext)
}
addFilter("Title", queryMetadata.Title)
addFilter("Filenames", queryMetadata.Filename)
addFilter("Extensions", queryMetadata.Extension)
addFilter("Languages", queryMetadata.Language)
addFilter("Topics", queryMetadata.Topic)
if queryMetadata.Default != "" {
query = queryMetadata.Default
var fields []string
for _, f := range strings.Split(config.C.SearchDefault, ",") {
f = strings.TrimSpace(f)
if f == "all" {
fields = AllSearchFields
break
}
if indexField, ok := SearchFieldMap[f]; ok {
fields = append(fields, indexField)
}
}
if len(fields) > 0 {
for _, f := range fields {
if f == "Content" {
fields = append(fields, "ContentSplit")
break
}
}
searchRequest.AttributesToSearchOn = fields
}
} else {
// Fuzzy-matchable fields become part of the query
var queryParts []string
var searchFields []string
if queryMetadata.Content != "" {
queryParts = append(queryParts, queryMetadata.Content)
searchFields = append(searchFields, "Content", "ContentSplit")
}
if queryMetadata.Title != "" {
queryParts = append(queryParts, queryMetadata.Title)
searchFields = append(searchFields, "Title")
}
if queryMetadata.Description != "" {
queryParts = append(queryParts, queryMetadata.Description)
searchFields = append(searchFields, "Description")
}
if queryMetadata.Filename != "" {
queryParts = append(queryParts, queryMetadata.Filename)
searchFields = append(searchFields, "Filenames")
}
query = strings.Join(queryParts, " ")
if len(searchFields) > 0 {
searchRequest.AttributesToSearchOn = searchFields
}
}
}
if len(filters) > 0 {
searchRequest.Filter = strings.Join(filters, " AND ")
}
response, err := (*atomicIndexer.Load()).(*MeiliIndexer).index.Search(query, searchRequest)
response, err := (*atomicIndexer.Load()).(*MeiliIndexer).index.Search(queryStr, searchRequest)
if err != nil {
log.Error().Err(err).Msg("Failed to search Meilisearch index")
return nil, 0, nil, err
}
gistIds := make([]uint, 0, len(response.Hits))
for _, hit := range response.Hits {
if gistIDRaw, ok := hit["GistID"]; ok {
@@ -232,9 +158,7 @@ func (i *MeiliIndexer) Search(queryMetadata SearchGistMetadata, userId uint, pag
var facetDist map[string]map[string]int
if err := json.Unmarshal(response.FacetDistribution, &facetDist); err == nil {
if facets, ok := facetDist["Languages"]; ok {
for lang, count := range facets {
languageCounts[strings.ToLower(lang)] += count
}
languageCounts = facets
}
}
}
@@ -242,30 +166,6 @@ func (i *MeiliIndexer) Search(queryMetadata SearchGistMetadata, userId uint, pag
return gistIds, uint64(response.EstimatedTotalHits), languageCounts, nil
}
func splitCamelCase(text string) string {
var result strings.Builder
runes := []rune(text)
for i := 0; i < len(runes); i++ {
r := runes[i]
if i > 0 {
prev := runes[i-1]
if unicode.IsUpper(r) {
if unicode.IsLower(prev) || unicode.IsDigit(prev) {
result.WriteRune(' ')
} else if unicode.IsUpper(prev) && i+1 < len(runes) && unicode.IsLower(runes[i+1]) {
result.WriteRune(' ')
}
} else if unicode.IsDigit(r) && !unicode.IsDigit(prev) {
result.WriteRune(' ')
} else if !unicode.IsDigit(r) && unicode.IsDigit(prev) {
result.WriteRune(' ')
}
}
result.WriteRune(r)
}
return result.String()
}
func escapeFilterValue(value string) string {
escaped := strings.ReplaceAll(value, "\\", "\\\\")
escaped = strings.ReplaceAll(escaped, "\"", "\\\"")

View File

@@ -1,88 +0,0 @@
package index
import (
"fmt"
"os"
"strconv"
"testing"
"github.com/meilisearch/meilisearch-go"
"github.com/rs/zerolog"
)
// syncMeiliIndexer wraps MeiliIndexer to make Add/Remove synchronous for tests.
type syncMeiliIndexer struct {
*MeiliIndexer
}
func (s *syncMeiliIndexer) Add(gist *Gist) error {
if gist == nil {
return fmt.Errorf("failed to add nil gist to index")
}
doc := &meiliGist{
Gist: *gist,
ContentSplit: splitCamelCase(gist.Content),
}
primaryKey := "GistID"
taskInfo, err := s.index.AddDocuments(doc, &meilisearch.DocumentOptions{PrimaryKey: &primaryKey})
if err != nil {
return err
}
_, err = s.client.WaitForTask(taskInfo.TaskUID, 0)
return err
}
func (s *syncMeiliIndexer) Remove(gistID uint) error {
taskInfo, err := s.index.DeleteDocument(strconv.Itoa(int(gistID)), nil)
if err != nil {
return err
}
_, err = s.client.WaitForTask(taskInfo.TaskUID, 0)
return err
}
func setupMeiliIndexer(t *testing.T) (Indexer, func()) {
zerolog.SetGlobalLevel(zerolog.Disabled)
t.Helper()
host := os.Getenv("OG_TEST_MEILI_HOST")
if host == "" {
host = "http://localhost:47700"
}
apiKey := os.Getenv("OG_TEST_MEILI_API_KEY")
indexName := fmt.Sprintf("test_%d", os.Getpid())
inner := NewMeiliIndexer(host, apiKey, indexName)
err := inner.Init()
if err != nil {
t.Skipf("MeiliSearch not available at %s: %v", host, err)
}
wrapped := &syncMeiliIndexer{MeiliIndexer: inner}
// Store the inner MeiliIndexer in atomicIndexer, because MeiliIndexer.Search
// type-asserts the global to *MeiliIndexer.
var idx Indexer = inner
atomicIndexer.Store(&idx)
cleanup := func() {
atomicIndexer.Store(nil)
inner.Reset()
inner.Close()
}
return wrapped, cleanup
}
func TestMeiliAddAndSearch(t *testing.T) { testAddAndSearch(t, setupMeiliIndexer) }
func TestMeiliAccessControl(t *testing.T) { testAccessControl(t, setupMeiliIndexer) }
func TestMeiliMetadataFilters(t *testing.T) { testMetadataFilters(t, setupMeiliIndexer) }
func TestMeiliAllFieldSearch(t *testing.T) { testAllFieldSearch(t, setupMeiliIndexer) }
func TestMeiliFuzzySearch(t *testing.T) { testFuzzySearch(t, setupMeiliIndexer) }
func TestMeiliContentSearch(t *testing.T) { testContentSearch(t, setupMeiliIndexer) }
func TestMeiliPagination(t *testing.T) { testPagination(t, setupMeiliIndexer) }
func TestMeiliLanguageFacets(t *testing.T) { testLanguageFacets(t, setupMeiliIndexer) }
func TestMeiliMetadataOnlySearch(t *testing.T) { testMetadataOnlySearch(t, setupMeiliIndexer) }
func TestMeiliTitleFuzzySearch(t *testing.T) { testTitleFuzzySearch(t, setupMeiliIndexer) }
func TestMeiliMultiLanguageFacets(t *testing.T) { testMultiLanguageFacets(t, setupMeiliIndexer) }

View File

@@ -164,18 +164,6 @@ func AllGists(ctx *context.Context) error {
return ctx.Html("all.html")
}
// Search handles the search page for gists.
//
// It takes a query parameter "q" which is a search query in the format:
// "user:username title:title description:description filename:filename language:language topic:topic"
//
// It also takes a page parameter "page" which is the page number to display.
//
// It returns an error if the search query is invalid or if the page number is invalid.
//
// It returns the search results as a list of rendered gists, along with the total number of results, the languages found, and the search query.
//
// The search results are paginated, with 10 results per page.
func Search(ctx *context.Context) error {
var err error
@@ -183,7 +171,7 @@ func Search(ctx *context.Context) error {
Query: ctx.QueryParam("q"),
}
metadata := handlers.ParseSearchQueryStr(ctx.QueryParam("q"))
content, meta := handlers.ParseSearchQueryStr(ctx.QueryParam("q"))
pageInt := handlers.GetPage(ctx)
var currentUserId uint
@@ -194,18 +182,14 @@ func Search(ctx *context.Context) error {
currentUserId = 0
}
// Search gists in the index and fetch the gists IDs from the database
gistsIds, nbHits, langs, err := index.SearchGists(index.SearchGistMetadata{
Username: metadata["user"],
Title: metadata["title"],
Description: metadata["description"],
Filename: metadata["filename"],
Extension: metadata["extension"],
Language: metadata["language"],
Topic: metadata["topic"],
Content: metadata["content"],
All: metadata["all"],
Default: metadata["default"],
gistsIds, nbHits, langs, err := index.SearchGists(content, index.SearchGistMetadata{
Username: meta["user"],
Title: meta["title"],
Filename: meta["filename"],
Extension: meta["extension"],
Language: meta["language"],
Topic: meta["topic"],
All: meta["all"],
}, currentUserId, pageInt)
if err != nil {
return ctx.ErrorRes(500, "Error searching gists", err)

View File

@@ -8,7 +8,7 @@ import (
"strings"
"github.com/gorilla/schema"
"github.com/rs/zerolog/log"
"github.com/thomiceli/opengist/internal/web/context"
)
@@ -119,16 +119,10 @@ func Paginate[T any](ctx *context.Context, data []*T, pageInt int, perPage int,
return nil
}
// ParseSearchQueryStr parses a search query string and returns a map of metadata.
// The query string is split into words and each word is checked if it contains a colon (:).
// If a word contains a colon, it is split into a key-value pair and added to the metadata map.
// If a word does not contain a colon, it is added to an "all" key in the metadata map.
// The "all" key is used to search all fields in the index.
// The function returns the metadata map.
func ParseSearchQueryStr(query string) map[string]string {
func ParseSearchQueryStr(query string) (string, map[string]string) {
words := strings.Fields(query)
metadata := make(map[string]string)
var allFieldsBuilder strings.Builder
var contentBuilder strings.Builder
for _, word := range words {
if strings.Contains(word, ":") {
@@ -139,18 +133,10 @@ func ParseSearchQueryStr(query string) map[string]string {
metadata[key] = value
}
} else {
// Add to content search by default
allFieldsBuilder.WriteString(word + " ")
contentBuilder.WriteString(word + " ")
}
}
// Set the default search field
allContent := strings.TrimSpace(allFieldsBuilder.String())
if allContent != "" {
metadata["default"] = allContent
}
log.Debug().Msgf("Metadata: %v", metadata)
return metadata
content := strings.TrimSpace(contentBuilder.String())
return content, metadata
}

View File

@@ -147,10 +147,7 @@ func (s *Server) setFuncMap() {
return dict, nil
},
"addMetadataToSearchQuery": func(input, key, value string) string {
metadata := handlers.ParseSearchQueryStr(input)
// extract free-text content (stored under "all") and remove it from metadata
content := metadata["all"]
delete(metadata, "all")
content, metadata := handlers.ParseSearchQueryStr(input)
metadata[key] = value

40
package-lock.json generated
View File

@@ -8,20 +8,20 @@
"devDependencies": {
"@catppuccin/highlightjs": "^1.0.1",
"@codemirror/commands": "^6.10.1",
"@codemirror/lang-javascript": "^6.2.5",
"@codemirror/lang-javascript": "^6.2.4",
"@codemirror/language": "^6.12.2",
"@codemirror/state": "^6.5.4",
"@codemirror/text": "^0.19.6",
"@codemirror/view": "^6.39.16",
"@codemirror/view": "^6.39.11",
"@tailwindcss/forms": "^0.5.11",
"@tailwindcss/typography": "^0.5.19",
"@tailwindcss/vite": "^4.2.1",
"codemirror": "^6.0.2",
"github-markdown-css": "^5.9.0",
"github-markdown-css": "^5.8.1",
"highlight.js": "^11.11.1",
"jdenticon": "^3.3.0",
"katex": "^0.16.38",
"marked": "^17.0.4",
"katex": "^0.16.33",
"marked": "^17.0.3",
"nodemon": "^3.1.11",
"pdfobject": "^2.3.1",
"tailwindcss": "^4.2.1",
@@ -78,9 +78,9 @@
}
},
"node_modules/@codemirror/lang-javascript": {
"version": "6.2.5",
"resolved": "https://registry.npmjs.org/@codemirror/lang-javascript/-/lang-javascript-6.2.5.tgz",
"integrity": "sha512-zD4e5mS+50htS7F+TYjBPsiIFGanfVqg4HyUz6WNFikgOPf2BgKlx+TQedI1w6n/IqRBVBbBWmGFdLB/7uxO4A==",
"version": "6.2.4",
"resolved": "https://registry.npmjs.org/@codemirror/lang-javascript/-/lang-javascript-6.2.4.tgz",
"integrity": "sha512-0WVmhp1QOqZ4Rt6GlVGwKJN3KW7Xh4H2q8ZZNGZaP6lRdxXJzmjm4FqvmOojVj6khWJHIb9sp7U/72W7xQgqAA==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -150,9 +150,9 @@
"license": "MIT"
},
"node_modules/@codemirror/view": {
"version": "6.39.16",
"resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.39.16.tgz",
"integrity": "sha512-m6S22fFpKtOWhq8HuhzsI1WzUP/hB9THbDj0Tl5KX4gbO6Y91hwBl7Yky33NdvB6IffuRFiBxf1R8kJMyXmA4Q==",
"version": "6.39.11",
"resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.39.11.tgz",
"integrity": "sha512-bWdeR8gWM87l4DB/kYSF9A+dVackzDb/V56Tq7QVrQ7rn86W0rgZFtlL3g3pem6AeGcb9NQNoy3ao4WpW4h5tQ==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -1636,9 +1636,9 @@
}
},
"node_modules/github-markdown-css": {
"version": "5.9.0",
"resolved": "https://registry.npmjs.org/github-markdown-css/-/github-markdown-css-5.9.0.tgz",
"integrity": "sha512-tmT5sY+zvg2302XLYEfH2mtkViIM1SWf2nvYoF5N1ZsO0V6B2qZTiw3GOzw4vpjLygK/KG35qRlPFweHqfzz5w==",
"version": "5.8.1",
"resolved": "https://registry.npmjs.org/github-markdown-css/-/github-markdown-css-5.8.1.tgz",
"integrity": "sha512-8G+PFvqigBQSWLQjyzgpa2ThD9bo7+kDsriUIidGcRhXgmcaAWUIpCZf8DavJgc+xifjbCG+GvMyWr0XMXmc7g==",
"dev": true,
"license": "MIT",
"engines": {
@@ -1768,9 +1768,9 @@
}
},
"node_modules/katex": {
"version": "0.16.38",
"resolved": "https://registry.npmjs.org/katex/-/katex-0.16.38.tgz",
"integrity": "sha512-cjHooZUmIAUmDsHBN+1n8LaZdpmbj03LtYeYPyuYB7OuloiaeaV6N4LcfjcnHVzGWjVQmKrxxTrpDcmSzEZQwQ==",
"version": "0.16.33",
"resolved": "https://registry.npmjs.org/katex/-/katex-0.16.33.tgz",
"integrity": "sha512-q3N5u+1sY9Bu7T4nlXoiRBXWfwSefNGoKeOwekV+gw0cAXQlz2Ww6BLcmBxVDeXBMUDQv6fK5bcNaJLxob3ZQA==",
"dev": true,
"funding": [
"https://opencollective.com/katex",
@@ -2066,9 +2066,9 @@
}
},
"node_modules/marked": {
"version": "17.0.4",
"resolved": "https://registry.npmjs.org/marked/-/marked-17.0.4.tgz",
"integrity": "sha512-NOmVMM+KAokHMvjWmC5N/ZOvgmSWuqJB8FoYI019j4ogb/PeRMKoKIjReZ2w3376kkA8dSJIP8uD993Kxc0iRQ==",
"version": "17.0.3",
"resolved": "https://registry.npmjs.org/marked/-/marked-17.0.3.tgz",
"integrity": "sha512-jt1v2ObpyOKR8p4XaUJVk3YWRJ5n+i4+rjQopxvV32rSndTJXvIzuUdWWIy/1pFQMkQmvTXawzDNqOH/CUmx6A==",
"dev": true,
"license": "MIT",
"bin": {

View File

@@ -13,20 +13,20 @@
"devDependencies": {
"@catppuccin/highlightjs": "^1.0.1",
"@codemirror/commands": "^6.10.1",
"@codemirror/lang-javascript": "^6.2.5",
"@codemirror/lang-javascript": "^6.2.4",
"@codemirror/language": "^6.12.2",
"@codemirror/state": "^6.5.4",
"@codemirror/text": "^0.19.6",
"@codemirror/view": "^6.39.16",
"@codemirror/view": "^6.39.11",
"@tailwindcss/forms": "^0.5.11",
"@tailwindcss/typography": "^0.5.19",
"@tailwindcss/vite": "^4.2.1",
"codemirror": "^6.0.2",
"github-markdown-css": "^5.9.0",
"github-markdown-css": "^5.8.1",
"highlight.js": "^11.11.1",
"jdenticon": "^3.3.0",
"katex": "^0.16.38",
"marked": "^17.0.4",
"katex": "^0.16.33",
"marked": "^17.0.3",
"nodemon": "^3.1.11",
"pdfobject": "^2.3.1",
"tailwindcss": "^4.2.1",

View File

@@ -116,12 +116,10 @@
<div class="p-4 text-xs space-y-1">
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">user:thomas</code> {{ .locale.Tr "gist.search.help.user" }}</p>
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">title:mygist</code> {{ .locale.Tr "gist.search.help.title" }}</p>
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">description:sync</code> {{ .locale.Tr "gist.search.help.description" }}</p>
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">filename:myfile.txt</code> {{ .locale.Tr "gist.search.help.filename" }}</p>
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">extension:yml</code> {{ .locale.Tr "gist.search.help.extension" }}</p>
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">language:go</code> {{ .locale.Tr "gist.search.help.language" }}</p>
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">topic:homelab</code> {{ .locale.Tr "gist.search.help.topic" }}</p>
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">all:systemctl</code> {{ .locale.Tr "gist.search.help.all" }}</p>
</div>
</div>
</div>

277
test.md Normal file
View File

@@ -0,0 +1,277 @@
---
description: Testing handler and middleware
slug: /testing
sidebar_position: 13
---
# Testing
## Testing Handler
`GET` `/users/:id`
Handler below retrieves user by id from the database. If user is not found it returns
`404` error with a message.
### CreateUser
`POST` `/users`
- Accepts JSON payload
- On success `201 - Created`
- On error `500 - Internal Server Error`
### GetUser
`GET` `/users/:email`
- On success `200 - OK`
- On error `404 - Not Found` if user is not found otherwise `500 - Internal Server Error`
`handler.go`
```go
package handler
import (
"net/http"
"github.com/labstack/echo/v5"
)
type (
User struct {
Name string `json:"name" form:"name"`
Email string `json:"email" form:"email"`
}
handler struct {
db map[string]*User
}
)
func (h *handler) createUser(c *echo.Context) error {
u := new(User)
if err := c.Bind(u); err != nil {
return err
}
return c.JSON(http.StatusCreated, u)
}
func (h *handler) getUser(c *echo.Context) error {
email := c.Param("email")
user := h.db[email]
if user == nil {
return echo.NewHTTPError(http.StatusNotFound, "user not found")
}
return c.JSON(http.StatusOK, user)
}
```
`handler_test.go`
```go
package handler
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/labstack/echo/v5"
"github.com/labstack/echo/v5/echotest"
"github.com/stretchr/testify/assert"
)
var (
mockDB = map[string]*User{
"jon@labstack.com": &User{"Jon Snow", "jon@labstack.com"},
}
userJSON = `{"name":"Jon Snow","email":"jon@labstack.com"}`
)
func TestCreateUser(t *testing.T) {
// Setup
e := echo.New()
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(userJSON))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
h := &controller{mockDB}
// Assertions
if assert.NoError(t, h.createUser(c)) {
assert.Equal(t, http.StatusCreated, rec.Code)
assert.Equal(t, userJSON, rec.Body.String())
}
}
// Same test as above but using `echotest` package helpers
func TestCreateUserWithEchoTest(t *testing.T) {
c, rec := echotest.ContextConfig{
Headers: map[string][]string{
echo.HeaderContentType: {echo.MIMEApplicationJSON},
},
JSONBody: []byte(`{"name":"Jon Snow","email":"jon@labstack.com"}`),
}.ToContextRecorder(t)
h := &controller{mockDB}
// Assertions
if assert.NoError(t, h.createUser(c)) {
assert.Equal(t, http.StatusCreated, rec.Code)
assert.Equal(t, userJSON+"\n", rec.Body.String())
}
}
// Same test as above but even shorter
func TestCreateUserWithEchoTest2(t *testing.T) {
h := &controller{mockDB}
rec := echotest.ContextConfig{
Headers: map[string][]string{
echo.HeaderContentType: {echo.MIMEApplicationJSON},
},
JSONBody: []byte(`{"name":"Jon Snow","email":"jon@labstack.com"}`),
}.ServeWithHandler(t, h.createUser)
assert.Equal(t, http.StatusCreated, rec.Code)
assert.Equal(t, userJSON+"\n", rec.Body.String())
}
func TestGetUser(t *testing.T) {
// Setup
e := echo.New()
req := httptest.NewRequest(http.MethodGet, "/", nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
c.SetPath("/users/:email")
c.SetPathValues(echo.PathValues{
{Name: "email", Value: "jon@labstack.com"},
})
h := &controller{mockDB}
// Assertions
if assert.NoError(t, h.getUser(c)) {
assert.Equal(t, http.StatusOK, rec.Code)
assert.Equal(t, userJSON, rec.Body.String())
}
}
func TestGetUserWithEchoTest(t *testing.T) {
c, rec := echotest.ContextConfig{
PathValues: echo.PathValues{
{Name: "email", Value: "jon@labstack.com"},
},
Headers: map[string][]string{
echo.HeaderContentType: {echo.MIMEApplicationJSON},
},
JSONBody: []byte(userJSON),
}.ToContextRecorder(t)
h := &controller{mockDB}
// Assertions
if assert.NoError(t, h.getUser(c)) {
assert.Equal(t, http.StatusOK, rec.Code)
assert.Equal(t, userJSON+"\n", rec.Body.String())
}
}
```
### Using Form Payload
```go
// import "net/url"
f := make(url.Values)
f.Set("name", "Jon Snow")
f.Set("email", "jon@labstack.com")
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(f.Encode()))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationForm)
```
Multipart form payload:
```go
func TestContext_MultipartForm(t *testing.T) {
testConf := echotest.ContextConfig{
MultipartForm: &echotest.MultipartForm{
Fields: map[string]string{
"key": "value",
},
Files: []echotest.MultipartFormFile{
{
Fieldname: "file",
Filename: "test.json",
Content: echotest.LoadBytes(t, "testdata/test.json"),
},
},
},
}
c := testConf.ToContext(t)
assert.Equal(t, "value", c.FormValue("key"))
assert.Equal(t, http.MethodPost, c.Request().Method)
assert.Equal(t, true, strings.HasPrefix(c.Request().Header.Get(echo.HeaderContentType), "multipart/form-data; boundary="))
fv, err := c.FormFile("file")
if err != nil {
t.Fatal(err)
}
assert.Equal(t, "test.json", fv.Filename)
}
```
### Setting Path Params
```go
c.SetPathValues(echo.PathValues{
{Name: "id", Value: "1"},
{Name: "email", Value: "jon@labstack.com"},
})
```
### Setting Query Params
```go
// import "net/url"
q := make(url.Values)
q.Set("email", "jon@labstack.com")
req := httptest.NewRequest(http.MethodGet, "/?"+q.Encode(), nil)
```
## Testing Middleware
```go
func TestCreateUserWithEchoTest2(t *testing.T) {
handler := func(c *echo.Context) error {
return c.JSON(http.StatusTeapot, fmt.Sprintf("email: %s", c.Param("email")))
}
middleware := func(next echo.HandlerFunc) echo.HandlerFunc {
return func(c *echo.Context) error {
c.Set("user_id", int64(1234))
return next(c)
}
}
c, rec := echotest.ContextConfig{
PathValues: echo.PathValues{{Name: "email", Value: "jon@labstack.com"}},
}.ToContextRecorder(t)
err := middleware(handler)(c)
if err != nil {
t.Fatal(err)
}
// check that middleware set the value
userID, err := echo.ContextGet[int64](c, "user_id")
assert.NoError(t, err)
assert.Equal(t, int64(1234), userID)
// check that handler returned the correct response
assert.Equal(t, http.StatusTeapot, rec.Code)
}
```
For now you can look into built-in middleware [test cases](https://github.com/labstack/echo/tree/master/middleware).

158
test2.md Normal file
View File

@@ -0,0 +1,158 @@
---
description: Testing handler and middleware
slug: /testing
sidebar_position: 13
---
# Testing
## Testing Handler
`GET` `/users/:id`
Handler below retrieves user by id from the database. If user is not found it returns
`404` error with a message.
### CreateUser
`POST` `/users`
- Accepts JSON payload
- On success `201 - Created`
- On error `500 - Internal Server Error`
### GetUser
`GET` `/users/:email`
- On success `200 - OK`
- On error `404 - Not Found` if user is not found otherwise `500 - Internal Server Error`
`handler.go`
```go
package handler
import (
"net/http"
"github.com/labstack/echo/v4"
)
type (
User struct {
Name string `json:"name" form:"name"`
Email string `json:"email" form:"email"`
}
handler struct {
db map[string]*User
}
)
func (h *handler) createUser(c echo.Context) error {
u := new(User)
if err := c.Bind(u); err != nil {
return err
}
return c.JSON(http.StatusCreated, u)
}
func (h *handler) getUser(c echo.Context) error {
email := c.Param("email")
user := h.db[email]
if user == nil {
return echo.NewHTTPError(http.StatusNotFound, "user not found")
}
return c.JSON(http.StatusOK, user)
}
```
`handler_test.go`
```go
package handler
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/labstack/echo/v4"
"github.com/stretchr/testify/assert"
)
var (
mockDB = map[string]*User{
"jon@labstack.com": &User{"Jon Snow", "jon@labstack.com"},
}
userJSON = `{"name":"Jon Snow","email":"jon@labstack.com"}`
)
func TestCreateUser(t *testing.T) {
// Setup
e := echo.New()
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(userJSON))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
h := &handler{mockDB}
// Assertions
if assert.NoError(t, h.createUser(c)) {
assert.Equal(t, http.StatusCreated, rec.Code)
assert.Equal(t, userJSON, rec.Body.String())
}
}
func TestGetUser(t *testing.T) {
// Setup
e := echo.New()
req := httptest.NewRequest(http.MethodGet, "/", nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
c.SetPath("/users/:email")
c.SetParamNames("email")
c.SetParamValues("jon@labstack.com")
h := &handler{mockDB}
// Assertions
if assert.NoError(t, h.getUser(c)) {
assert.Equal(t, http.StatusOK, rec.Code)
assert.Equal(t, userJSON, rec.Body.String())
}
}
```
### Using Form Payload
```go
// import "net/url"
f := make(url.Values)
f.Set("name", "Jon Snow")
f.Set("email", "jon@labstack.com")
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(f.Encode()))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationForm)
```
### Setting Path Params
```go
c.SetParamNames("id", "email")
c.SetParamValues("1", "jon@labstack.com")
```
### Setting Query Params
```go
// import "net/url"
q := make(url.Values)
q.Set("email", "jon@labstack.com")
req := httptest.NewRequest(http.MethodGet, "/?"+q.Encode(), nil)
```
## Testing Middleware
*TBD*
For now you can look into built-in middleware [test cases](https://github.com/labstack/echo/tree/master/middleware).