Compare commits

..

1 Commits

Author SHA1 Message Date
Anonymous
69b9b215d6 Translated using Weblate (Ukrainian)
Currently translated at 69.4% (243 of 350 strings)

Translation: Opengist/Opengist
Translate-URL: http://tr.opengist.io/projects/_/opengist/uk/
2026-03-09 23:52:08 +00:00
23 changed files with 2242 additions and 1350 deletions

View File

@@ -83,18 +83,6 @@ jobs:
--health-interval 10s
--health-timeout 5s
--health-retries 5
meilisearch:
image: getmeili/meilisearch:latest
ports:
- 47700:7700
env:
MEILI_NO_ANALYTICS: true
MEILI_ENV: development
options: >-
--health-cmd "curl -sf http://localhost:7700/health"
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- name: Checkout
uses: actions/checkout@v6
@@ -106,8 +94,6 @@ jobs:
- name: Run tests
run: make test TEST_DB_TYPE=${{ matrix.database }}
env:
OG_TEST_MEILI_HOST: http://localhost:47700
test:
name: Test

View File

@@ -42,7 +42,7 @@ jobs:
- name: Docker meta
id: meta
uses: docker/metadata-action@v6
uses: docker/metadata-action@v5
with:
images: |
ghcr.io/thomiceli/opengist
@@ -54,26 +54,26 @@ jobs:
type=semver,pattern={{version}}
- name: Set up QEMU
uses: docker/setup-qemu-action@v4
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Login to GitHub Container Registry
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to Docker Hub
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and push
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64,linux/arm64

View File

@@ -32,10 +32,6 @@ index.meili.host:
# Set the API key for the Meiliseach server
index.meili.api-key:
# Set the default search fields. Can contain multiple fields (e.g., `content,username`).
# Fields: content,user,title,description,filename,extension,language,topic. Default: content
search.default: content
# Default branch name used by Opengist when initializing Git repositories.
# If not set, uses the Git default branch name. See https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup#_new_default_branch
git.default-branch:

View File

@@ -15,7 +15,6 @@ aside: false
| index | OG_INDEX | `bleve` | Define the code indexer (either `bleve`, `meilisearch`, or empty for no index). |
| index.meili.host | OG_MEILI_HOST | none | Set the host for the Meiliseach server. |
| index.meili.api-key | OG_MEILI_API_KEY | none | Set the API key for the Meiliseach server. |
| search.default | OG_SEARCH_DEFAULT | `content` | Set the default search fields. Can contain multiple fields (e.g., `content,username`). Fields: `content,user,title,description,filename,extension,language,topic`. |
| git.default-branch | OG_GIT_DEFAULT_BRANCH | none | Default branch name used by Opengist when initializing Git repositories. If not set, uses the Git default branch name. More info [here](https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup#_new_default_branch) |
| sqlite.journal-mode | OG_SQLITE_JOURNAL_MODE | `WAL` | Set the journal mode for SQLite. More info [here](https://www.sqlite.org/pragma.html#pragma_journal_mode) |
| http.host | OG_HTTP_HOST | `0.0.0.0` | The host on which the HTTP server should bind. Use an IP address for network binding. Use a path for Unix socket binding (e.g. /run/opengist.sock) |

View File

@@ -38,12 +38,11 @@ type config struct {
DBUri string `yaml:"db-uri" env:"OG_DB_URI"`
DBFilename string `yaml:"db-filename" env:"OG_DB_FILENAME"` // deprecated
IndexEnabled bool `yaml:"index.enabled" env:"OG_INDEX_ENABLED"` // deprecated
Index string `yaml:"index" env:"OG_INDEX"`
BleveDirname string `yaml:"index.dirname" env:"OG_INDEX_DIRNAME"` // deprecated
MeiliHost string `yaml:"index.meili.host" env:"OG_MEILI_HOST"`
MeiliAPIKey string `yaml:"index.meili.api-key" env:"OG_MEILI_API_KEY"`
SearchDefault string `yaml:"search.default" env:"OG_SEARCH_DEFAULT"`
IndexEnabled bool `yaml:"index.enabled" env:"OG_INDEX_ENABLED"` // deprecated
Index string `yaml:"index" env:"OG_INDEX"`
BleveDirname string `yaml:"index.dirname" env:"OG_INDEX_DIRNAME"` // deprecated
MeiliHost string `yaml:"index.meili.host" env:"OG_MEILI_HOST"`
MeiliAPIKey string `yaml:"index.meili.api-key" env:"OG_MEILI_API_KEY"`
GitDefaultBranch string `yaml:"git.default-branch" env:"OG_GIT_DEFAULT_BRANCH"`
@@ -112,7 +111,6 @@ func configWithDefaults() (*config, error) {
c.OpengistHome = ""
c.DBUri = "opengist.db"
c.Index = "bleve"
c.SearchDefault = "content"
c.SqliteJournalMode = "WAL"

View File

@@ -817,19 +817,18 @@ func (gist *Gist) ToIndexedGist() (*index.Gist, error) {
}
indexedGist := &index.Gist{
GistID: gist.ID,
UserID: gist.UserID,
Visibility: gist.Private.Uint(),
Username: gist.User.Username,
Description: gist.Description,
Title: gist.Title,
Content: wholeContent,
Filenames: fileNames,
Extensions: exts,
Languages: langs,
Topics: topics,
CreatedAt: gist.CreatedAt,
UpdatedAt: gist.UpdatedAt,
GistID: gist.ID,
UserID: gist.UserID,
Visibility: gist.Private.Uint(),
Username: gist.User.Username,
Title: gist.Title,
Content: wholeContent,
Filenames: fileNames,
Extensions: exts,
Languages: langs,
Topics: topics,
CreatedAt: gist.CreatedAt,
UpdatedAt: gist.UpdatedAt,
}
return indexedGist, nil

View File

@@ -88,12 +88,10 @@ gist.search.found: gists found
gist.search.no-results: No gists found
gist.search.help.user: gists created by user
gist.search.help.title: gists with given title
gist.search.help.description: gists with given description
gist.search.help.filename: gists having files with given name
gist.search.help.extension: gists having files with given extension
gist.search.help.language: gists having files with given language
gist.search.help.topic: gists with given topic
gist.search.help.all: search all fields
gist.search.placeholder.title: Title
gist.search.placeholder.visibility: Visibility
gist.search.placeholder.public: Public

View File

@@ -77,7 +77,7 @@ gist.list.all-from: Всі gists від %s
gist.search.found: gists знайдено
gist.search.no-results: Не знайдено gists
gist.search.help.user: gists створені користувачем
gist.search.help.title: gists з наданим ім'ям
gist.search.help.title: gists з наданим ім'ям
gist.search.help.filename: gists мають файли з наданим ім'ям
gist.search.help.extension: gists мають файли з наданим розширенням
gist.search.help.language: gists мають файли з наданою мовою
@@ -266,4 +266,4 @@ validation.should-only-contain-alphanumeric-characters-and-dashes: Поле %s
validation.not-enough: Недостатньо %s
validation.invalid: Недійсний %s
html.title.admin-panel: Панель адміністратора
html.title.admin-panel: Панель адміністратора

View File

@@ -5,18 +5,15 @@ import (
"fmt"
"os"
"strconv"
"strings"
"github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
"github.com/blevesearch/bleve/v2/analysis/token/camelcase"
"github.com/blevesearch/bleve/v2/analysis/token/length"
"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
"github.com/blevesearch/bleve/v2/analysis/token/unicodenorm"
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
"github.com/blevesearch/bleve/v2/search/query"
"github.com/rs/zerolog/log"
"github.com/thomiceli/opengist/internal/config"
)
type BleveIndexer struct {
@@ -57,9 +54,14 @@ func (i *BleveIndexer) open() (bleve.Index, error) {
return nil, err
}
docMapping := bleve.NewDocumentMapping()
docMapping.AddFieldMappingsAt("GistID", bleve.NewNumericFieldMapping())
docMapping.AddFieldMappingsAt("UserID", bleve.NewNumericFieldMapping())
docMapping.AddFieldMappingsAt("Visibility", bleve.NewNumericFieldMapping())
docMapping.AddFieldMappingsAt("Content", bleve.NewTextFieldMapping())
mapping := bleve.NewIndexMapping()
// Token filters
if err = mapping.AddCustomTokenFilter("unicodeNormalize", map[string]any{
"type": unicodenorm.Name,
"form": unicodenorm.NFC,
@@ -67,74 +69,16 @@ func (i *BleveIndexer) open() (bleve.Index, error) {
return nil, err
}
if err = mapping.AddCustomTokenFilter("lengthMin2", map[string]interface{}{
"type": length.Name,
"min": 2.0,
}); err != nil {
return nil, err
}
// Analyzer: split mode (camelCase splitting for partial search)
// "CPUCard" -> ["cpu", "card"]
if err = mapping.AddCustomAnalyzer("codeSplit", map[string]interface{}{
if err = mapping.AddCustomAnalyzer("gistAnalyser", map[string]interface{}{
"type": custom.Name,
"char_filters": []string{},
"tokenizer": unicode.Name,
"token_filters": []string{"unicodeNormalize", camelcase.Name, lowercase.Name, "lengthMin2"},
"token_filters": []string{"unicodeNormalize", camelcase.Name, lowercase.Name},
}); err != nil {
return nil, err
}
// Analyzer: exact mode (no camelCase splitting for full-word search)
// "CPUCard" -> ["cpucard"]
if err = mapping.AddCustomAnalyzer("codeExact", map[string]interface{}{
"type": custom.Name,
"char_filters": []string{},
"tokenizer": unicode.Name,
"token_filters": []string{"unicodeNormalize", lowercase.Name},
}); err != nil {
return nil, err
}
// Analyzer: keyword with lowercase (for Languages - single token, no splitting)
if err = mapping.AddCustomAnalyzer("lowercaseKeyword", map[string]interface{}{
"type": custom.Name,
"char_filters": []string{},
"tokenizer": "single",
"token_filters": []string{lowercase.Name},
}); err != nil {
return nil, err
}
// Document mapping
docMapping := bleve.NewDocumentMapping()
docMapping.AddFieldMappingsAt("GistID", bleve.NewNumericFieldMapping())
docMapping.AddFieldMappingsAt("UserID", bleve.NewNumericFieldMapping())
docMapping.AddFieldMappingsAt("Visibility", bleve.NewNumericFieldMapping())
// Content: dual indexing (exact + split)
// "Content" uses the property name so Bleve resolves its analyzer correctly
contentExact := bleve.NewTextFieldMapping()
contentExact.Name = "Content"
contentExact.Analyzer = "codeExact"
contentExact.Store = false
contentExact.IncludeTermVectors = true
contentSplit := bleve.NewTextFieldMapping()
contentSplit.Name = "ContentSplit"
contentSplit.Analyzer = "codeSplit"
contentSplit.Store = false
contentSplit.IncludeTermVectors = true
docMapping.AddFieldMappingsAt("Content", contentExact, contentSplit)
// Languages: keyword analyzer (preserves as single token)
languageFieldMapping := bleve.NewTextFieldMapping()
languageFieldMapping.Analyzer = "lowercaseKeyword"
docMapping.AddFieldMappingsAt("Languages", languageFieldMapping)
// All other text fields use codeSplit as default
docMapping.DefaultAnalyzer = "codeSplit"
docMapping.DefaultAnalyzer = "gistAnalyser"
mapping.DefaultMapping = docMapping
return bleve.New(i.path, mapping)
@@ -172,111 +116,18 @@ func (i *BleveIndexer) Remove(gistID uint) error {
return (*atomicIndexer.Load()).(*BleveIndexer).index.Delete(strconv.Itoa(int(gistID)))
}
// Search returns a list of Gist IDs that match the given search metadata.
// The method returns an error if any.
//
// The queryMetadata parameter is used to filter the search results.
// For example, passing a non-empty Username will search for gists whose
// username matches the given string.
//
// If the "All" field in queryMetadata is non-empty, the method will
// search across all metadata fields with OR logic. Otherwise, the method
// will add each metadata field with AND logic.
//
// The page parameter is used to paginate the search results.
// The method returns the total number of search results in the second return
// value.
//
// The third return value is a map of language counts for the search results.
// The language counts are computed by asking Bleve to return the top 10
// facets for the "Languages" field.
func (i *BleveIndexer) Search(metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
func (i *BleveIndexer) Search(queryStr string, queryMetadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
var err error
var indexerQuery query.Query = bleve.NewMatchAllQuery()
// Query factory
factoryQuery := func(field, value string) query.Query {
query := bleve.NewMatchPhraseQuery(value)
query.SetField(field)
return query
}
// Exact search
addQuery := func(field, value string) {
if value != "" && value != "." {
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, factoryQuery(field, value))
}
}
// Query factory for text fields: exact match boosted + match query + prefix
factoryTextQuery := func(field, value string) query.Query {
exact := bleve.NewMatchPhraseQuery(value)
exact.SetField(field)
exact.SetBoost(2.0)
fuzzy := bleve.NewMatchQuery(value)
fuzzy.SetField(field)
fuzzy.SetFuzziness(1)
fuzzy.SetOperator(query.MatchQueryOperatorAnd)
queries := []query.Query{exact, fuzzy}
if len([]rune(value)) >= 2 {
prefix := bleve.NewPrefixQuery(strings.ToLower(value))
prefix.SetField(field)
prefix.SetBoost(1.5)
queries = append(queries, prefix)
}
if len([]rune(value)) >= 4 {
wildcard := bleve.NewWildcardQuery("*" + strings.ToLower(value) + "*")
wildcard.SetField(field)
wildcard.SetBoost(0.5)
queries = append(queries, wildcard)
}
return bleve.NewDisjunctionQuery(queries...)
}
// Query factory for Content: searches both exact (Content) and split (ContentSplit) fields
factoryContentQuery := func(value string) query.Query {
// Exact field (no camelCase split): matches "cpucard"
exactMatch := bleve.NewMatchQuery(value)
exactMatch.SetField("Content")
exactMatch.SetOperator(query.MatchQueryOperatorAnd)
exactMatch.SetBoost(2.0)
// Split field (camelCase split): matches "cpu", "card"
splitMatch := bleve.NewMatchQuery(value)
splitMatch.SetField("ContentSplit")
splitMatch.SetFuzziness(1)
splitMatch.SetOperator(query.MatchQueryOperatorAnd)
splitMatch.SetBoost(1.0)
queries := []query.Query{exactMatch, splitMatch}
if len([]rune(value)) >= 2 {
prefix := bleve.NewPrefixQuery(strings.ToLower(value))
prefix.SetField("Content")
prefix.SetBoost(1.5)
queries = append(queries, prefix)
}
if len([]rune(value)) >= 4 {
wildcard := bleve.NewWildcardQuery("*" + strings.ToLower(value) + "*")
wildcard.SetField("Content")
wildcard.SetBoost(0.5)
queries = append(queries, wildcard)
}
return bleve.NewDisjunctionQuery(queries...)
}
// Text field search
addTextQuery := func(field, value string) {
if value != "" && value != "." {
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, factoryTextQuery(field, value))
}
var indexerQuery query.Query
if queryStr != "" {
// Use match query with fuzzy matching for more flexible content search
contentQuery := bleve.NewMatchQuery(queryStr)
contentQuery.SetField("Content")
contentQuery.SetFuzziness(2)
indexerQuery = contentQuery
} else {
contentQuery := bleve.NewMatchAllQuery()
indexerQuery = contentQuery
}
// Visibility filtering: show public gists (Visibility=0) OR user's own gists
@@ -292,62 +143,48 @@ func (i *BleveIndexer) Search(metadata SearchGistMetadata, userId uint, page int
accessQuery := bleve.NewDisjunctionQuery(publicQuery, userIdQuery)
indexerQuery = bleve.NewConjunctionQuery(accessQuery, indexerQuery)
buildFieldQuery := func(field, value string) query.Query {
switch field {
case "Content":
return factoryContentQuery(value)
case "Title", "Description", "Filenames":
return factoryTextQuery(field, value)
case "Extensions":
return factoryQuery(field, "."+value)
default: // Username, Languages, Topics
return factoryQuery(field, value)
}
}
// Handle "All" field - search across all metadata fields with OR logic
if metadata.All != "" {
allQueries := make([]query.Query, 0, len(AllSearchFields))
for _, field := range AllSearchFields {
allQueries = append(allQueries, buildFieldQuery(field, metadata.All))
if queryMetadata.All != "" {
allQueries := make([]query.Query, 0)
// Create match phrase queries for each field
fields := []struct {
field string
value string
}{
{"Username", queryMetadata.All},
{"Title", queryMetadata.All},
{"Extensions", "." + queryMetadata.All},
{"Filenames", queryMetadata.All},
{"Languages", queryMetadata.All},
{"Topics", queryMetadata.All},
}
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, bleve.NewDisjunctionQuery(allQueries...))
for _, f := range fields {
q := bleve.NewMatchPhraseQuery(f.value)
q.FieldVal = f.field
allQueries = append(allQueries, q)
}
// Combine all field queries with OR (disjunction)
allDisjunction := bleve.NewDisjunctionQuery(allQueries...)
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, allDisjunction)
} else {
// Original behavior: add each metadata field with AND logic
addQuery("Username", metadata.Username)
addTextQuery("Title", metadata.Title)
addTextQuery("Description", metadata.Description)
addQuery("Extensions", "."+metadata.Extension)
addTextQuery("Filenames", metadata.Filename)
addQuery("Languages", metadata.Language)
addQuery("Topics", metadata.Topic)
if metadata.Content != "" {
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, factoryContentQuery(metadata.Content))
addQuery := func(field, value string) {
if value != "" && value != "." {
q := bleve.NewMatchPhraseQuery(value)
q.FieldVal = field
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, q)
}
}
// Handle default search fields from config with OR logic
if metadata.Default != "" {
var fields []string
for _, f := range strings.Split(config.C.SearchDefault, ",") {
f = strings.TrimSpace(f)
if f == "all" {
fields = AllSearchFields
break
}
if indexField, ok := SearchFieldMap[f]; ok {
fields = append(fields, indexField)
}
}
if len(fields) == 1 {
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, buildFieldQuery(fields[0], metadata.Default))
} else if len(fields) > 1 {
defaultQueries := make([]query.Query, 0, len(fields))
for _, field := range fields {
defaultQueries = append(defaultQueries, buildFieldQuery(field, metadata.Default))
}
indexerQuery = bleve.NewConjunctionQuery(indexerQuery, bleve.NewDisjunctionQuery(defaultQueries...))
}
}
addQuery("Username", queryMetadata.Username)
addQuery("Title", queryMetadata.Title)
addQuery("Extensions", "."+queryMetadata.Extension)
addQuery("Filenames", queryMetadata.Filename)
addQuery("Languages", queryMetadata.Language)
addQuery("Topics", queryMetadata.Topic)
}
languageFacet := bleve.NewFacetRequest("Languages", 10)
@@ -360,8 +197,6 @@ func (i *BleveIndexer) Search(metadata SearchGistMetadata, userId uint, page int
s.Fields = []string{"GistID"}
s.IncludeLocations = false
log.Debug().Interface("searchRequest", s).Msg("Bleve search request")
results, err := (*atomicIndexer.Load()).(*BleveIndexer).index.Search(s)
if err != nil {
return nil, 0, nil, err

View File

@@ -4,31 +4,33 @@ import (
"os"
"path/filepath"
"testing"
"github.com/rs/zerolog"
"github.com/stretchr/testify/require"
)
// setupBleveIndexer creates a new BleveIndexer for testing
func setupBleveIndexer(t *testing.T) (Indexer, func()) {
zerolog.SetGlobalLevel(zerolog.Disabled)
func setupBleveIndexer(t *testing.T) (*BleveIndexer, func()) {
t.Helper()
// Create a temporary directory for the test index
tmpDir, err := os.MkdirTemp("", "bleve-test-*")
require.NoError(t, err)
if err != nil {
t.Fatalf("Failed to create temp directory: %v", err)
}
indexPath := filepath.Join(tmpDir, "test.index")
indexer := NewBleveIndexer(indexPath)
// Initialize the indexer
err = indexer.Init()
if err != nil {
os.RemoveAll(tmpDir)
t.Fatalf("Failed to initialize BleveIndexer: %v", err)
}
// Store in the global atomicIndexer since Add/Remove use it
var idx Indexer = indexer
atomicIndexer.Store(&idx)
// Return cleanup function
cleanup := func() {
atomicIndexer.Store(nil)
indexer.Close()
@@ -38,50 +40,123 @@ func setupBleveIndexer(t *testing.T) (Indexer, func()) {
return indexer, cleanup
}
func TestBleveAddAndSearch(t *testing.T) { testAddAndSearch(t, setupBleveIndexer) }
func TestBleveAccessControl(t *testing.T) { testAccessControl(t, setupBleveIndexer) }
func TestBleveMetadataFilters(t *testing.T) { testMetadataFilters(t, setupBleveIndexer) }
func TestBleveAllFieldSearch(t *testing.T) { testAllFieldSearch(t, setupBleveIndexer) }
func TestBleveFuzzySearch(t *testing.T) { testFuzzySearch(t, setupBleveIndexer) }
func TestBleveContentSearch(t *testing.T) { testContentSearch(t, setupBleveIndexer) }
func TestBlevePagination(t *testing.T) { testPagination(t, setupBleveIndexer) }
func TestBleveLanguageFacets(t *testing.T) { testLanguageFacets(t, setupBleveIndexer) }
func TestBleveWildcardSearch(t *testing.T) { testWildcardSearch(t, setupBleveIndexer) }
func TestBleveMetadataOnlySearch(t *testing.T) { testMetadataOnlySearch(t, setupBleveIndexer) }
func TestBleveTitleFuzzySearch(t *testing.T) { testTitleFuzzySearch(t, setupBleveIndexer) }
func TestBleveMultiLanguageFacets(t *testing.T) { testMultiLanguageFacets(t, setupBleveIndexer) }
func TestBleveIndexerAddGist(t *testing.T) {
indexer, cleanup := setupBleveIndexer(t)
defer cleanup()
func TestBlevePersistence(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "bleve-persist-test-*")
require.NoError(t, err)
testIndexerAddGist(t, indexer)
}
func TestBleveIndexerAllFieldSearch(t *testing.T) {
indexer, cleanup := setupBleveIndexer(t)
defer cleanup()
testIndexerAllFieldSearch(t, indexer)
}
func TestBleveIndexerFuzzySearch(t *testing.T) {
indexer, cleanup := setupBleveIndexer(t)
defer cleanup()
testIndexerFuzzySearch(t, indexer)
}
func TestBleveIndexerSearchBasic(t *testing.T) {
indexer, cleanup := setupBleveIndexer(t)
defer cleanup()
testIndexerSearchBasic(t, indexer)
}
func TestBleveIndexerPagination(t *testing.T) {
indexer, cleanup := setupBleveIndexer(t)
defer cleanup()
testIndexerPagination(t, indexer)
}
// TestBleveIndexerInitAndClose tests Bleve-specific initialization and closing
func TestBleveIndexerInitAndClose(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "bleve-init-test-*")
if err != nil {
t.Fatalf("Failed to create temp directory: %v", err)
}
defer os.RemoveAll(tmpDir)
indexPath := filepath.Join(tmpDir, "test.index")
indexer := NewBleveIndexer(indexPath)
// Create and populate index
indexer1 := NewBleveIndexer(indexPath)
require.NoError(t, indexer1.Init())
// Test initialization
err = indexer.Init()
if err != nil {
t.Fatalf("Failed to initialize BleveIndexer: %v", err)
}
var idx Indexer = indexer1
atomicIndexer.Store(&idx)
if indexer.index == nil {
t.Fatal("Expected index to be initialized, got nil")
}
g := newGist(1, 1, 0, "persistent data survives restart")
require.NoError(t, indexer1.Add(g))
// Test closing
indexer.Close()
indexer1.Close()
atomicIndexer.Store(nil)
// Reopen at same path
// Test reopening the same index
indexer2 := NewBleveIndexer(indexPath)
require.NoError(t, indexer2.Init())
err = indexer2.Init()
if err != nil {
t.Fatalf("Failed to reopen BleveIndexer: %v", err)
}
defer indexer2.Close()
idx = indexer2
atomicIndexer.Store(&idx)
defer atomicIndexer.Store(nil)
ids, total, _, err := indexer2.Search(SearchGistMetadata{Content: "persistent"}, 1, 1)
require.NoError(t, err)
require.Equal(t, uint64(1), total, "data should survive close+reopen")
require.Equal(t, uint(1), ids[0])
if indexer2.index == nil {
t.Fatal("Expected reopened index to be initialized, got nil")
}
}
// TestBleveIndexerUnicodeSearch tests that Unicode content can be indexed and searched
func TestBleveIndexerUnicodeSearch(t *testing.T) {
indexer, cleanup := setupBleveIndexer(t)
defer cleanup()
// Add a gist with Unicode content
gist := &Gist{
GistID: 100,
UserID: 100,
Visibility: 0,
Username: "testuser",
Title: "Unicode Test",
Content: "Hello world with unicode characters: café résumé naïve",
Filenames: []string{"test.txt"},
Extensions: []string{".txt"},
Languages: []string{"Text"},
Topics: []string{"unicode"},
CreatedAt: 1234567890,
UpdatedAt: 1234567890,
}
err := indexer.Add(gist)
if err != nil {
t.Fatalf("Failed to add gist: %v", err)
}
// Search for unicode content
gistIDs, total, _, err := indexer.Search("café", SearchGistMetadata{}, 100, 1)
if err != nil {
t.Fatalf("Search failed: %v", err)
}
if total == 0 {
t.Skip("Unicode search may require specific index configuration")
return
}
found := false
for _, id := range gistIDs {
if id == 100 {
found = true
break
}
}
if !found {
t.Log("Unicode gist not found in search results, but other results were returned")
}
}

View File

@@ -1,43 +1,26 @@
package index
var AllSearchFields = []string{"Username", "Title", "Description", "Filenames", "Extensions", "Languages", "Topics", "Content"}
var SearchFieldMap = map[string]string{
"user": "Username",
"title": "Title",
"description": "Description",
"filename": "Filenames",
"extension": "Extensions",
"language": "Languages",
"topic": "Topics",
"content": "Content",
}
type Gist struct {
GistID uint
UserID uint
Visibility uint
Username string
Description string
Title string
Content string
Filenames []string
Extensions []string
Languages []string
Topics []string
CreatedAt int64
UpdatedAt int64
GistID uint
UserID uint
Visibility uint
Username string
Title string
Content string
Filenames []string
Extensions []string
Languages []string
Topics []string
CreatedAt int64
UpdatedAt int64
}
type SearchGistMetadata struct {
Username string
Title string
Description string
Content string
Filename string
Extension string
Language string
Topic string
All string
Default string
Username string
Title string
Filename string
Extension string
Language string
Topic string
All string
}

View File

@@ -17,7 +17,7 @@ type Indexer interface {
Reset() error
Add(gist *Gist) error
Remove(gistID uint) error
Search(metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error)
Search(query string, metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error)
}
type IndexerType string
@@ -125,11 +125,7 @@ func RemoveFromIndex(gistID uint) error {
return (*idx).Remove(gistID)
}
// SearchGists returns a list of Gist IDs that match the given search metadata.
// If the indexer is not enabled, it returns nil, 0, nil, nil.
// If the indexer is not initialized, it returns nil, 0, nil, fmt.Errorf("indexer is not initialized").
// The function returns an error if any.
func SearchGists(metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
func SearchGists(query string, metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
if !IndexEnabled() {
return nil, 0, nil, nil
}
@@ -139,7 +135,7 @@ func SearchGists(metadata SearchGistMetadata, userId uint, page int) ([]uint, ui
return nil, 0, nil, fmt.Errorf("indexer is not initialized")
}
return (*idx).Search(metadata, userId, page)
return (*idx).Search(query, metadata, userId, page)
}
func DepreactionIndexDirname() {

File diff suppressed because it is too large Load Diff

View File

@@ -6,11 +6,9 @@ import (
"fmt"
"strconv"
"strings"
"unicode"
"github.com/meilisearch/meilisearch-go"
"github.com/rs/zerolog/log"
"github.com/thomiceli/opengist/internal/config"
)
type MeiliIndexer struct {
@@ -52,25 +50,23 @@ func (i *MeiliIndexer) open() (meilisearch.IndexManager, error) {
i.client = meilisearch.New(i.host, meilisearch.WithAPIKey(i.apikey))
indexResult, err := i.client.GetIndex(i.indexName)
if indexResult == nil || err != nil {
_, err = i.client.CreateIndex(&meilisearch.IndexConfig{
Uid: i.indexName,
PrimaryKey: "GistID",
})
if err != nil {
return nil, err
}
if indexResult != nil && err == nil {
return indexResult.IndexManager, nil
}
_, err = i.client.CreateIndex(&meilisearch.IndexConfig{
Uid: i.indexName,
PrimaryKey: "GistID",
})
if err != nil {
return nil, err
}
_, _ = i.client.Index(i.indexName).UpdateSettings(&meilisearch.Settings{
FilterableAttributes: []string{"GistID", "UserID", "Visibility", "Username", "Extensions", "Languages", "Topics"},
SearchableAttributes: []string{"Content", "ContentSplit", "Username", "Title", "Description", "Filenames", "Extensions", "Languages", "Topics"},
RankingRules: []string{"words", "typo", "proximity", "attribute", "sort", "exactness"},
TypoTolerance: &meilisearch.TypoTolerance{
Enabled: true,
DisableOnNumbers: true,
MinWordSizeForTypos: meilisearch.MinWordSizeForTypos{OneTypo: 4, TwoTypos: 10},
},
FilterableAttributes: []string{"GistID", "UserID", "Visibility", "Username", "Title", "Filenames", "Extensions", "Languages", "Topics"},
DisplayedAttributes: []string{"GistID"},
SearchableAttributes: []string{"Content", "Username", "Title", "Filenames", "Extensions", "Languages", "Topics"},
RankingRules: []string{"words"},
})
return i.client.Index(i.indexName), nil
@@ -99,21 +95,12 @@ func (i *MeiliIndexer) Close() {
i.client = nil
}
type meiliGist struct {
Gist
ContentSplit string
}
func (i *MeiliIndexer) Add(gist *Gist) error {
if gist == nil {
return errors.New("failed to add nil gist to index")
}
doc := &meiliGist{
Gist: *gist,
ContentSplit: splitCamelCase(gist.Content),
}
primaryKey := "GistID"
_, err := (*atomicIndexer.Load()).(*MeiliIndexer).index.AddDocuments(doc, &meilisearch.DocumentOptions{PrimaryKey: &primaryKey})
_, err := (*atomicIndexer.Load()).(*MeiliIndexer).index.AddDocuments(gist, &meilisearch.DocumentOptions{PrimaryKey: &primaryKey})
return err
}
@@ -122,14 +109,13 @@ func (i *MeiliIndexer) Remove(gistID uint) error {
return err
}
func (i *MeiliIndexer) Search(queryMetadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
func (i *MeiliIndexer) Search(queryStr string, queryMetadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
searchRequest := &meilisearch.SearchRequest{
Offset: int64((page - 1) * 10),
Limit: 11,
AttributesToRetrieve: []string{"GistID", "Languages"},
Facets: []string{"Languages"},
AttributesToSearchOn: []string{"Content", "ContentSplit"},
MatchingStrategy: meilisearch.All,
AttributesToSearchOn: []string{"Content"},
}
var filters []string
@@ -140,83 +126,23 @@ func (i *MeiliIndexer) Search(queryMetadata SearchGistMetadata, userId uint, pag
filters = append(filters, fmt.Sprintf("%s = \"%s\"", field, escapeFilterValue(value)))
}
}
var query string
if queryMetadata.All != "" {
query = queryMetadata.All
searchRequest.AttributesToSearchOn = append(AllSearchFields, "ContentSplit")
} else {
// Exact-match fields stay as filters
addFilter("Username", queryMetadata.Username)
if queryMetadata.Extension != "" {
ext := queryMetadata.Extension
if !strings.HasPrefix(ext, ".") {
ext = "." + ext
}
addFilter("Extensions", ext)
}
addFilter("Languages", queryMetadata.Language)
addFilter("Topics", queryMetadata.Topic)
if queryMetadata.Default != "" {
query = queryMetadata.Default
var fields []string
for _, f := range strings.Split(config.C.SearchDefault, ",") {
f = strings.TrimSpace(f)
if f == "all" {
fields = AllSearchFields
break
}
if indexField, ok := SearchFieldMap[f]; ok {
fields = append(fields, indexField)
}
}
if len(fields) > 0 {
for _, f := range fields {
if f == "Content" {
fields = append(fields, "ContentSplit")
break
}
}
searchRequest.AttributesToSearchOn = fields
}
} else {
// Fuzzy-matchable fields become part of the query
var queryParts []string
var searchFields []string
if queryMetadata.Content != "" {
queryParts = append(queryParts, queryMetadata.Content)
searchFields = append(searchFields, "Content", "ContentSplit")
}
if queryMetadata.Title != "" {
queryParts = append(queryParts, queryMetadata.Title)
searchFields = append(searchFields, "Title")
}
if queryMetadata.Description != "" {
queryParts = append(queryParts, queryMetadata.Description)
searchFields = append(searchFields, "Description")
}
if queryMetadata.Filename != "" {
queryParts = append(queryParts, queryMetadata.Filename)
searchFields = append(searchFields, "Filenames")
}
query = strings.Join(queryParts, " ")
if len(searchFields) > 0 {
searchRequest.AttributesToSearchOn = searchFields
}
}
}
addFilter("Username", queryMetadata.Username)
addFilter("Title", queryMetadata.Title)
addFilter("Filenames", queryMetadata.Filename)
addFilter("Extensions", queryMetadata.Extension)
addFilter("Languages", queryMetadata.Language)
addFilter("Topics", queryMetadata.Topic)
if len(filters) > 0 {
searchRequest.Filter = strings.Join(filters, " AND ")
}
response, err := (*atomicIndexer.Load()).(*MeiliIndexer).index.Search(query, searchRequest)
response, err := (*atomicIndexer.Load()).(*MeiliIndexer).index.Search(queryStr, searchRequest)
if err != nil {
log.Error().Err(err).Msg("Failed to search Meilisearch index")
return nil, 0, nil, err
}
gistIds := make([]uint, 0, len(response.Hits))
for _, hit := range response.Hits {
if gistIDRaw, ok := hit["GistID"]; ok {
@@ -232,9 +158,7 @@ func (i *MeiliIndexer) Search(queryMetadata SearchGistMetadata, userId uint, pag
var facetDist map[string]map[string]int
if err := json.Unmarshal(response.FacetDistribution, &facetDist); err == nil {
if facets, ok := facetDist["Languages"]; ok {
for lang, count := range facets {
languageCounts[strings.ToLower(lang)] += count
}
languageCounts = facets
}
}
}
@@ -242,30 +166,6 @@ func (i *MeiliIndexer) Search(queryMetadata SearchGistMetadata, userId uint, pag
return gistIds, uint64(response.EstimatedTotalHits), languageCounts, nil
}
func splitCamelCase(text string) string {
var result strings.Builder
runes := []rune(text)
for i := 0; i < len(runes); i++ {
r := runes[i]
if i > 0 {
prev := runes[i-1]
if unicode.IsUpper(r) {
if unicode.IsLower(prev) || unicode.IsDigit(prev) {
result.WriteRune(' ')
} else if unicode.IsUpper(prev) && i+1 < len(runes) && unicode.IsLower(runes[i+1]) {
result.WriteRune(' ')
}
} else if unicode.IsDigit(r) && !unicode.IsDigit(prev) {
result.WriteRune(' ')
} else if !unicode.IsDigit(r) && unicode.IsDigit(prev) {
result.WriteRune(' ')
}
}
result.WriteRune(r)
}
return result.String()
}
func escapeFilterValue(value string) string {
escaped := strings.ReplaceAll(value, "\\", "\\\\")
escaped = strings.ReplaceAll(escaped, "\"", "\\\"")

View File

@@ -1,88 +0,0 @@
package index
import (
"fmt"
"os"
"strconv"
"testing"
"github.com/meilisearch/meilisearch-go"
"github.com/rs/zerolog"
)
// syncMeiliIndexer wraps MeiliIndexer to make Add/Remove synchronous for tests.
type syncMeiliIndexer struct {
*MeiliIndexer
}
func (s *syncMeiliIndexer) Add(gist *Gist) error {
if gist == nil {
return fmt.Errorf("failed to add nil gist to index")
}
doc := &meiliGist{
Gist: *gist,
ContentSplit: splitCamelCase(gist.Content),
}
primaryKey := "GistID"
taskInfo, err := s.index.AddDocuments(doc, &meilisearch.DocumentOptions{PrimaryKey: &primaryKey})
if err != nil {
return err
}
_, err = s.client.WaitForTask(taskInfo.TaskUID, 0)
return err
}
func (s *syncMeiliIndexer) Remove(gistID uint) error {
taskInfo, err := s.index.DeleteDocument(strconv.Itoa(int(gistID)), nil)
if err != nil {
return err
}
_, err = s.client.WaitForTask(taskInfo.TaskUID, 0)
return err
}
func setupMeiliIndexer(t *testing.T) (Indexer, func()) {
zerolog.SetGlobalLevel(zerolog.Disabled)
t.Helper()
host := os.Getenv("OG_TEST_MEILI_HOST")
if host == "" {
host = "http://localhost:47700"
}
apiKey := os.Getenv("OG_TEST_MEILI_API_KEY")
indexName := fmt.Sprintf("test_%d", os.Getpid())
inner := NewMeiliIndexer(host, apiKey, indexName)
err := inner.Init()
if err != nil {
t.Skipf("MeiliSearch not available at %s: %v", host, err)
}
wrapped := &syncMeiliIndexer{MeiliIndexer: inner}
// Store the inner MeiliIndexer in atomicIndexer, because MeiliIndexer.Search
// type-asserts the global to *MeiliIndexer.
var idx Indexer = inner
atomicIndexer.Store(&idx)
cleanup := func() {
atomicIndexer.Store(nil)
inner.Reset()
inner.Close()
}
return wrapped, cleanup
}
func TestMeiliAddAndSearch(t *testing.T) { testAddAndSearch(t, setupMeiliIndexer) }
func TestMeiliAccessControl(t *testing.T) { testAccessControl(t, setupMeiliIndexer) }
func TestMeiliMetadataFilters(t *testing.T) { testMetadataFilters(t, setupMeiliIndexer) }
func TestMeiliAllFieldSearch(t *testing.T) { testAllFieldSearch(t, setupMeiliIndexer) }
func TestMeiliFuzzySearch(t *testing.T) { testFuzzySearch(t, setupMeiliIndexer) }
func TestMeiliContentSearch(t *testing.T) { testContentSearch(t, setupMeiliIndexer) }
func TestMeiliPagination(t *testing.T) { testPagination(t, setupMeiliIndexer) }
func TestMeiliLanguageFacets(t *testing.T) { testLanguageFacets(t, setupMeiliIndexer) }
func TestMeiliMetadataOnlySearch(t *testing.T) { testMetadataOnlySearch(t, setupMeiliIndexer) }
func TestMeiliTitleFuzzySearch(t *testing.T) { testTitleFuzzySearch(t, setupMeiliIndexer) }
func TestMeiliMultiLanguageFacets(t *testing.T) { testMultiLanguageFacets(t, setupMeiliIndexer) }

View File

@@ -164,18 +164,6 @@ func AllGists(ctx *context.Context) error {
return ctx.Html("all.html")
}
// Search handles the search page for gists.
//
// It takes a query parameter "q" which is a search query in the format:
// "user:username title:title description:description filename:filename language:language topic:topic"
//
// It also takes a page parameter "page" which is the page number to display.
//
// It returns an error if the search query is invalid or if the page number is invalid.
//
// It returns the search results as a list of rendered gists, along with the total number of results, the languages found, and the search query.
//
// The search results are paginated, with 10 results per page.
func Search(ctx *context.Context) error {
var err error
@@ -183,7 +171,7 @@ func Search(ctx *context.Context) error {
Query: ctx.QueryParam("q"),
}
metadata := handlers.ParseSearchQueryStr(ctx.QueryParam("q"))
content, meta := handlers.ParseSearchQueryStr(ctx.QueryParam("q"))
pageInt := handlers.GetPage(ctx)
var currentUserId uint
@@ -194,18 +182,14 @@ func Search(ctx *context.Context) error {
currentUserId = 0
}
// Search gists in the index and fetch the gists IDs from the database
gistsIds, nbHits, langs, err := index.SearchGists(index.SearchGistMetadata{
Username: metadata["user"],
Title: metadata["title"],
Description: metadata["description"],
Filename: metadata["filename"],
Extension: metadata["extension"],
Language: metadata["language"],
Topic: metadata["topic"],
Content: metadata["content"],
All: metadata["all"],
Default: metadata["default"],
gistsIds, nbHits, langs, err := index.SearchGists(content, index.SearchGistMetadata{
Username: meta["user"],
Title: meta["title"],
Filename: meta["filename"],
Extension: meta["extension"],
Language: meta["language"],
Topic: meta["topic"],
All: meta["all"],
}, currentUserId, pageInt)
if err != nil {
return ctx.ErrorRes(500, "Error searching gists", err)

View File

@@ -8,7 +8,7 @@ import (
"strings"
"github.com/gorilla/schema"
"github.com/rs/zerolog/log"
"github.com/thomiceli/opengist/internal/web/context"
)
@@ -119,16 +119,10 @@ func Paginate[T any](ctx *context.Context, data []*T, pageInt int, perPage int,
return nil
}
// ParseSearchQueryStr parses a search query string and returns a map of metadata.
// The query string is split into words and each word is checked if it contains a colon (:).
// If a word contains a colon, it is split into a key-value pair and added to the metadata map.
// If a word does not contain a colon, it is added to an "all" key in the metadata map.
// The "all" key is used to search all fields in the index.
// The function returns the metadata map.
func ParseSearchQueryStr(query string) map[string]string {
func ParseSearchQueryStr(query string) (string, map[string]string) {
words := strings.Fields(query)
metadata := make(map[string]string)
var allFieldsBuilder strings.Builder
var contentBuilder strings.Builder
for _, word := range words {
if strings.Contains(word, ":") {
@@ -139,18 +133,10 @@ func ParseSearchQueryStr(query string) map[string]string {
metadata[key] = value
}
} else {
// Add to content search by default
allFieldsBuilder.WriteString(word + " ")
contentBuilder.WriteString(word + " ")
}
}
// Set the default search field
allContent := strings.TrimSpace(allFieldsBuilder.String())
if allContent != "" {
metadata["default"] = allContent
}
log.Debug().Msgf("Metadata: %v", metadata)
return metadata
content := strings.TrimSpace(contentBuilder.String())
return content, metadata
}

View File

@@ -147,10 +147,7 @@ func (s *Server) setFuncMap() {
return dict, nil
},
"addMetadataToSearchQuery": func(input, key, value string) string {
metadata := handlers.ParseSearchQueryStr(input)
// extract free-text content (stored under "all") and remove it from metadata
content := metadata["all"]
delete(metadata, "all")
content, metadata := handlers.ParseSearchQueryStr(input)
metadata[key] = value

40
package-lock.json generated
View File

@@ -8,20 +8,20 @@
"devDependencies": {
"@catppuccin/highlightjs": "^1.0.1",
"@codemirror/commands": "^6.10.1",
"@codemirror/lang-javascript": "^6.2.5",
"@codemirror/lang-javascript": "^6.2.4",
"@codemirror/language": "^6.12.2",
"@codemirror/state": "^6.5.4",
"@codemirror/text": "^0.19.6",
"@codemirror/view": "^6.39.16",
"@codemirror/view": "^6.39.11",
"@tailwindcss/forms": "^0.5.11",
"@tailwindcss/typography": "^0.5.19",
"@tailwindcss/vite": "^4.2.1",
"codemirror": "^6.0.2",
"github-markdown-css": "^5.9.0",
"github-markdown-css": "^5.8.1",
"highlight.js": "^11.11.1",
"jdenticon": "^3.3.0",
"katex": "^0.16.38",
"marked": "^17.0.4",
"katex": "^0.16.33",
"marked": "^17.0.3",
"nodemon": "^3.1.11",
"pdfobject": "^2.3.1",
"tailwindcss": "^4.2.1",
@@ -78,9 +78,9 @@
}
},
"node_modules/@codemirror/lang-javascript": {
"version": "6.2.5",
"resolved": "https://registry.npmjs.org/@codemirror/lang-javascript/-/lang-javascript-6.2.5.tgz",
"integrity": "sha512-zD4e5mS+50htS7F+TYjBPsiIFGanfVqg4HyUz6WNFikgOPf2BgKlx+TQedI1w6n/IqRBVBbBWmGFdLB/7uxO4A==",
"version": "6.2.4",
"resolved": "https://registry.npmjs.org/@codemirror/lang-javascript/-/lang-javascript-6.2.4.tgz",
"integrity": "sha512-0WVmhp1QOqZ4Rt6GlVGwKJN3KW7Xh4H2q8ZZNGZaP6lRdxXJzmjm4FqvmOojVj6khWJHIb9sp7U/72W7xQgqAA==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -150,9 +150,9 @@
"license": "MIT"
},
"node_modules/@codemirror/view": {
"version": "6.39.16",
"resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.39.16.tgz",
"integrity": "sha512-m6S22fFpKtOWhq8HuhzsI1WzUP/hB9THbDj0Tl5KX4gbO6Y91hwBl7Yky33NdvB6IffuRFiBxf1R8kJMyXmA4Q==",
"version": "6.39.11",
"resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.39.11.tgz",
"integrity": "sha512-bWdeR8gWM87l4DB/kYSF9A+dVackzDb/V56Tq7QVrQ7rn86W0rgZFtlL3g3pem6AeGcb9NQNoy3ao4WpW4h5tQ==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -1636,9 +1636,9 @@
}
},
"node_modules/github-markdown-css": {
"version": "5.9.0",
"resolved": "https://registry.npmjs.org/github-markdown-css/-/github-markdown-css-5.9.0.tgz",
"integrity": "sha512-tmT5sY+zvg2302XLYEfH2mtkViIM1SWf2nvYoF5N1ZsO0V6B2qZTiw3GOzw4vpjLygK/KG35qRlPFweHqfzz5w==",
"version": "5.8.1",
"resolved": "https://registry.npmjs.org/github-markdown-css/-/github-markdown-css-5.8.1.tgz",
"integrity": "sha512-8G+PFvqigBQSWLQjyzgpa2ThD9bo7+kDsriUIidGcRhXgmcaAWUIpCZf8DavJgc+xifjbCG+GvMyWr0XMXmc7g==",
"dev": true,
"license": "MIT",
"engines": {
@@ -1768,9 +1768,9 @@
}
},
"node_modules/katex": {
"version": "0.16.38",
"resolved": "https://registry.npmjs.org/katex/-/katex-0.16.38.tgz",
"integrity": "sha512-cjHooZUmIAUmDsHBN+1n8LaZdpmbj03LtYeYPyuYB7OuloiaeaV6N4LcfjcnHVzGWjVQmKrxxTrpDcmSzEZQwQ==",
"version": "0.16.33",
"resolved": "https://registry.npmjs.org/katex/-/katex-0.16.33.tgz",
"integrity": "sha512-q3N5u+1sY9Bu7T4nlXoiRBXWfwSefNGoKeOwekV+gw0cAXQlz2Ww6BLcmBxVDeXBMUDQv6fK5bcNaJLxob3ZQA==",
"dev": true,
"funding": [
"https://opencollective.com/katex",
@@ -2066,9 +2066,9 @@
}
},
"node_modules/marked": {
"version": "17.0.4",
"resolved": "https://registry.npmjs.org/marked/-/marked-17.0.4.tgz",
"integrity": "sha512-NOmVMM+KAokHMvjWmC5N/ZOvgmSWuqJB8FoYI019j4ogb/PeRMKoKIjReZ2w3376kkA8dSJIP8uD993Kxc0iRQ==",
"version": "17.0.3",
"resolved": "https://registry.npmjs.org/marked/-/marked-17.0.3.tgz",
"integrity": "sha512-jt1v2ObpyOKR8p4XaUJVk3YWRJ5n+i4+rjQopxvV32rSndTJXvIzuUdWWIy/1pFQMkQmvTXawzDNqOH/CUmx6A==",
"dev": true,
"license": "MIT",
"bin": {

View File

@@ -13,20 +13,20 @@
"devDependencies": {
"@catppuccin/highlightjs": "^1.0.1",
"@codemirror/commands": "^6.10.1",
"@codemirror/lang-javascript": "^6.2.5",
"@codemirror/lang-javascript": "^6.2.4",
"@codemirror/language": "^6.12.2",
"@codemirror/state": "^6.5.4",
"@codemirror/text": "^0.19.6",
"@codemirror/view": "^6.39.16",
"@codemirror/view": "^6.39.11",
"@tailwindcss/forms": "^0.5.11",
"@tailwindcss/typography": "^0.5.19",
"@tailwindcss/vite": "^4.2.1",
"codemirror": "^6.0.2",
"github-markdown-css": "^5.9.0",
"github-markdown-css": "^5.8.1",
"highlight.js": "^11.11.1",
"jdenticon": "^3.3.0",
"katex": "^0.16.38",
"marked": "^17.0.4",
"katex": "^0.16.33",
"marked": "^17.0.3",
"nodemon": "^3.1.11",
"pdfobject": "^2.3.1",
"tailwindcss": "^4.2.1",

View File

@@ -116,12 +116,10 @@
<div class="p-4 text-xs space-y-1">
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">user:thomas</code> {{ .locale.Tr "gist.search.help.user" }}</p>
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">title:mygist</code> {{ .locale.Tr "gist.search.help.title" }}</p>
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">description:sync</code> {{ .locale.Tr "gist.search.help.description" }}</p>
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">filename:myfile.txt</code> {{ .locale.Tr "gist.search.help.filename" }}</p>
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">extension:yml</code> {{ .locale.Tr "gist.search.help.extension" }}</p>
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">language:go</code> {{ .locale.Tr "gist.search.help.language" }}</p>
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">topic:homelab</code> {{ .locale.Tr "gist.search.help.topic" }}</p>
<p class="text-gray-400"><code class="text-slate-800 dark:text-slate-300 pr-1">all:systemctl</code> {{ .locale.Tr "gist.search.help.all" }}</p>
</div>
</div>
</div>

277
test.md Normal file
View File

@@ -0,0 +1,277 @@
---
description: Testing handler and middleware
slug: /testing
sidebar_position: 13
---
# Testing
## Testing Handler
`GET` `/users/:id`
Handler below retrieves user by id from the database. If user is not found it returns
`404` error with a message.
### CreateUser
`POST` `/users`
- Accepts JSON payload
- On success `201 - Created`
- On error `500 - Internal Server Error`
### GetUser
`GET` `/users/:email`
- On success `200 - OK`
- On error `404 - Not Found` if user is not found otherwise `500 - Internal Server Error`
`handler.go`
```go
package handler
import (
"net/http"
"github.com/labstack/echo/v5"
)
type (
User struct {
Name string `json:"name" form:"name"`
Email string `json:"email" form:"email"`
}
handler struct {
db map[string]*User
}
)
func (h *handler) createUser(c *echo.Context) error {
u := new(User)
if err := c.Bind(u); err != nil {
return err
}
return c.JSON(http.StatusCreated, u)
}
func (h *handler) getUser(c *echo.Context) error {
email := c.Param("email")
user := h.db[email]
if user == nil {
return echo.NewHTTPError(http.StatusNotFound, "user not found")
}
return c.JSON(http.StatusOK, user)
}
```
`handler_test.go`
```go
package handler
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/labstack/echo/v5"
"github.com/labstack/echo/v5/echotest"
"github.com/stretchr/testify/assert"
)
var (
mockDB = map[string]*User{
"jon@labstack.com": &User{"Jon Snow", "jon@labstack.com"},
}
userJSON = `{"name":"Jon Snow","email":"jon@labstack.com"}`
)
func TestCreateUser(t *testing.T) {
// Setup
e := echo.New()
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(userJSON))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
h := &controller{mockDB}
// Assertions
if assert.NoError(t, h.createUser(c)) {
assert.Equal(t, http.StatusCreated, rec.Code)
assert.Equal(t, userJSON, rec.Body.String())
}
}
// Same test as above but using `echotest` package helpers
func TestCreateUserWithEchoTest(t *testing.T) {
c, rec := echotest.ContextConfig{
Headers: map[string][]string{
echo.HeaderContentType: {echo.MIMEApplicationJSON},
},
JSONBody: []byte(`{"name":"Jon Snow","email":"jon@labstack.com"}`),
}.ToContextRecorder(t)
h := &controller{mockDB}
// Assertions
if assert.NoError(t, h.createUser(c)) {
assert.Equal(t, http.StatusCreated, rec.Code)
assert.Equal(t, userJSON+"\n", rec.Body.String())
}
}
// Same test as above but even shorter
func TestCreateUserWithEchoTest2(t *testing.T) {
h := &controller{mockDB}
rec := echotest.ContextConfig{
Headers: map[string][]string{
echo.HeaderContentType: {echo.MIMEApplicationJSON},
},
JSONBody: []byte(`{"name":"Jon Snow","email":"jon@labstack.com"}`),
}.ServeWithHandler(t, h.createUser)
assert.Equal(t, http.StatusCreated, rec.Code)
assert.Equal(t, userJSON+"\n", rec.Body.String())
}
func TestGetUser(t *testing.T) {
// Setup
e := echo.New()
req := httptest.NewRequest(http.MethodGet, "/", nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
c.SetPath("/users/:email")
c.SetPathValues(echo.PathValues{
{Name: "email", Value: "jon@labstack.com"},
})
h := &controller{mockDB}
// Assertions
if assert.NoError(t, h.getUser(c)) {
assert.Equal(t, http.StatusOK, rec.Code)
assert.Equal(t, userJSON, rec.Body.String())
}
}
func TestGetUserWithEchoTest(t *testing.T) {
c, rec := echotest.ContextConfig{
PathValues: echo.PathValues{
{Name: "email", Value: "jon@labstack.com"},
},
Headers: map[string][]string{
echo.HeaderContentType: {echo.MIMEApplicationJSON},
},
JSONBody: []byte(userJSON),
}.ToContextRecorder(t)
h := &controller{mockDB}
// Assertions
if assert.NoError(t, h.getUser(c)) {
assert.Equal(t, http.StatusOK, rec.Code)
assert.Equal(t, userJSON+"\n", rec.Body.String())
}
}
```
### Using Form Payload
```go
// import "net/url"
f := make(url.Values)
f.Set("name", "Jon Snow")
f.Set("email", "jon@labstack.com")
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(f.Encode()))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationForm)
```
Multipart form payload:
```go
func TestContext_MultipartForm(t *testing.T) {
testConf := echotest.ContextConfig{
MultipartForm: &echotest.MultipartForm{
Fields: map[string]string{
"key": "value",
},
Files: []echotest.MultipartFormFile{
{
Fieldname: "file",
Filename: "test.json",
Content: echotest.LoadBytes(t, "testdata/test.json"),
},
},
},
}
c := testConf.ToContext(t)
assert.Equal(t, "value", c.FormValue("key"))
assert.Equal(t, http.MethodPost, c.Request().Method)
assert.Equal(t, true, strings.HasPrefix(c.Request().Header.Get(echo.HeaderContentType), "multipart/form-data; boundary="))
fv, err := c.FormFile("file")
if err != nil {
t.Fatal(err)
}
assert.Equal(t, "test.json", fv.Filename)
}
```
### Setting Path Params
```go
c.SetPathValues(echo.PathValues{
{Name: "id", Value: "1"},
{Name: "email", Value: "jon@labstack.com"},
})
```
### Setting Query Params
```go
// import "net/url"
q := make(url.Values)
q.Set("email", "jon@labstack.com")
req := httptest.NewRequest(http.MethodGet, "/?"+q.Encode(), nil)
```
## Testing Middleware
```go
func TestCreateUserWithEchoTest2(t *testing.T) {
handler := func(c *echo.Context) error {
return c.JSON(http.StatusTeapot, fmt.Sprintf("email: %s", c.Param("email")))
}
middleware := func(next echo.HandlerFunc) echo.HandlerFunc {
return func(c *echo.Context) error {
c.Set("user_id", int64(1234))
return next(c)
}
}
c, rec := echotest.ContextConfig{
PathValues: echo.PathValues{{Name: "email", Value: "jon@labstack.com"}},
}.ToContextRecorder(t)
err := middleware(handler)(c)
if err != nil {
t.Fatal(err)
}
// check that middleware set the value
userID, err := echo.ContextGet[int64](c, "user_id")
assert.NoError(t, err)
assert.Equal(t, int64(1234), userID)
// check that handler returned the correct response
assert.Equal(t, http.StatusTeapot, rec.Code)
}
```
For now you can look into built-in middleware [test cases](https://github.com/labstack/echo/tree/master/middleware).

158
test2.md Normal file
View File

@@ -0,0 +1,158 @@
---
description: Testing handler and middleware
slug: /testing
sidebar_position: 13
---
# Testing
## Testing Handler
`GET` `/users/:id`
Handler below retrieves user by id from the database. If user is not found it returns
`404` error with a message.
### CreateUser
`POST` `/users`
- Accepts JSON payload
- On success `201 - Created`
- On error `500 - Internal Server Error`
### GetUser
`GET` `/users/:email`
- On success `200 - OK`
- On error `404 - Not Found` if user is not found otherwise `500 - Internal Server Error`
`handler.go`
```go
package handler
import (
"net/http"
"github.com/labstack/echo/v4"
)
type (
User struct {
Name string `json:"name" form:"name"`
Email string `json:"email" form:"email"`
}
handler struct {
db map[string]*User
}
)
func (h *handler) createUser(c echo.Context) error {
u := new(User)
if err := c.Bind(u); err != nil {
return err
}
return c.JSON(http.StatusCreated, u)
}
func (h *handler) getUser(c echo.Context) error {
email := c.Param("email")
user := h.db[email]
if user == nil {
return echo.NewHTTPError(http.StatusNotFound, "user not found")
}
return c.JSON(http.StatusOK, user)
}
```
`handler_test.go`
```go
package handler
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/labstack/echo/v4"
"github.com/stretchr/testify/assert"
)
var (
mockDB = map[string]*User{
"jon@labstack.com": &User{"Jon Snow", "jon@labstack.com"},
}
userJSON = `{"name":"Jon Snow","email":"jon@labstack.com"}`
)
func TestCreateUser(t *testing.T) {
// Setup
e := echo.New()
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(userJSON))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
h := &handler{mockDB}
// Assertions
if assert.NoError(t, h.createUser(c)) {
assert.Equal(t, http.StatusCreated, rec.Code)
assert.Equal(t, userJSON, rec.Body.String())
}
}
func TestGetUser(t *testing.T) {
// Setup
e := echo.New()
req := httptest.NewRequest(http.MethodGet, "/", nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
c.SetPath("/users/:email")
c.SetParamNames("email")
c.SetParamValues("jon@labstack.com")
h := &handler{mockDB}
// Assertions
if assert.NoError(t, h.getUser(c)) {
assert.Equal(t, http.StatusOK, rec.Code)
assert.Equal(t, userJSON, rec.Body.String())
}
}
```
### Using Form Payload
```go
// import "net/url"
f := make(url.Values)
f.Set("name", "Jon Snow")
f.Set("email", "jon@labstack.com")
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(f.Encode()))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationForm)
```
### Setting Path Params
```go
c.SetParamNames("id", "email")
c.SetParamValues("1", "jon@labstack.com")
```
### Setting Query Params
```go
// import "net/url"
q := make(url.Values)
q.Set("email", "jon@labstack.com")
req := httptest.NewRequest(http.MethodGet, "/?"+q.Encode(), nil)
```
## Testing Middleware
*TBD*
For now you can look into built-in middleware [test cases](https://github.com/labstack/echo/tree/master/middleware).