From efba783c565f6c4056492f75c06c7679d117aba1 Mon Sep 17 00:00:00 2001
From: Thomas Miceli <27960254+thomiceli@users.noreply.github.com>
Date: Wed, 19 Mar 2025 23:28:04 +0100
Subject: [PATCH] Add Meilisearch indexer (#444)
---
config.yml | 11 ++-
docs/configuration/cheat-sheet.md | 5 +-
go.mod | 5 +
go.sum | 17 +++-
internal/cli/main.go | 8 +-
internal/config/config.go | 10 +-
internal/db/gist.go | 10 +-
internal/index/bleve.go | 104 +++++++-------------
internal/index/gist.go | 2 +
internal/index/indexer.go | 136 +++++++++++++++++++++++++++
internal/index/meilisearch.go | 146 +++++++++++++++++++++++++++++
internal/web/handlers/gist/all.go | 8 +-
internal/web/handlers/gist/edit.go | 2 +
internal/web/server/renderer.go | 2 +-
internal/web/server/router.go | 2 +-
internal/web/test/server.go | 2 +-
templates/pages/admin_config.html | 5 +-
17 files changed, 373 insertions(+), 102 deletions(-)
create mode 100644 internal/index/indexer.go
create mode 100644 internal/index/meilisearch.go
diff --git a/config.yml b/config.yml
index 213067a..0a891bb 100644
--- a/config.yml
+++ b/config.yml
@@ -23,11 +23,14 @@ secret-key:
# MySQL/MariaDB: mysql://user:password@host:port/database
db-uri: opengist.db
-# Enable or disable the code search index (either `true` or `false`). Default: true
-index.enabled: true
+# Define the code indexer (either `bleve`, `meilisearch`, or empty for no index). Default: bleve
+index: bleve
-# Name of the directory where the code search index is stored. Default: opengist.index
-index.dirname: opengist.index
+# Set the host for the Meiliseach server
+index.meili.host:
+
+# Set the API key for the Meiliseach server
+index.meili.api-key:
# Default branch name used by Opengist when initializing Git repositories.
# If not set, uses the Git default branch name. See https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup#_new_default_branch
diff --git a/docs/configuration/cheat-sheet.md b/docs/configuration/cheat-sheet.md
index 51b9240..af8eb83 100644
--- a/docs/configuration/cheat-sheet.md
+++ b/docs/configuration/cheat-sheet.md
@@ -12,8 +12,9 @@ aside: false
| opengist-home | OG_OPENGIST_HOME | home directory | Path to the directory where Opengist stores its data. |
| secret-key | OG_SECRET_KEY | randomized 32 bytes | Secret key used for session store & encrypt MFA data on database. |
| db-uri | OG_DB_URI | `opengist.db` | URI of the database. |
-| index.enabled | OG_INDEX_ENABLED | `true` | Enable or disable the code search index (`true` or `false`) |
-| index.dirname | OG_INDEX_DIRNAME | `opengist.index` | Name of the directory where the code search index is stored. |
+| index | OG_INDEX | `bleve` | Define the code indexer (either `bleve`, `meilisearch`, or empty for no index). |
+| index.meili.host | OG_MEILI_HOST | none | Set the host for the Meiliseach server. |
+| index.meili.api-key | OG_MEILI_API_KEY | none | Set the API key for the Meiliseach server. |
| git.default-branch | OG_GIT_DEFAULT_BRANCH | none | Default branch name used by Opengist when initializing Git repositories. If not set, uses the Git default branch name. More info [here](https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup#_new_default_branch) |
| sqlite.journal-mode | OG_SQLITE_JOURNAL_MODE | `WAL` | Set the journal mode for SQLite. More info [here](https://www.sqlite.org/pragma.html#pragma_journal_mode) |
| http.host | OG_HTTP_HOST | `0.0.0.0` | The host on which the HTTP server should bind. |
diff --git a/go.mod b/go.mod
index 4c06e59..f57c35f 100644
--- a/go.mod
+++ b/go.mod
@@ -17,6 +17,7 @@ require (
github.com/labstack/echo-contrib v0.17.2
github.com/labstack/echo/v4 v4.13.3
github.com/markbates/goth v1.80.0
+ github.com/meilisearch/meilisearch-go v0.31.0
github.com/pquerna/otp v1.4.0
github.com/prometheus/client_golang v1.20.5
github.com/rs/zerolog v1.33.0
@@ -37,6 +38,7 @@ require (
require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/RoaringBitmap/roaring v1.9.4 // indirect
+ github.com/andybalholm/brotli v1.1.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.17.0 // indirect
github.com/blevesearch/bleve_index_api v1.1.13 // indirect
@@ -68,6 +70,7 @@ require (
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-sql-driver/mysql v1.8.1 // indirect
github.com/go-webauthn/x v0.1.15 // indirect
+ github.com/golang-jwt/jwt/v4 v4.5.1 // indirect
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
github.com/golang/geo v0.0.0-20230421003525-6adc56603217 // indirect
github.com/golang/protobuf v1.5.4 // indirect
@@ -80,11 +83,13 @@ require (
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
+ github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.17.11 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/labstack/gommon v0.4.2 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
+ github.com/mailru/easyjson v0.9.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
diff --git a/go.sum b/go.sum
index ffb0cc4..861ac1c 100644
--- a/go.sum
+++ b/go.sum
@@ -12,6 +12,8 @@ github.com/alecthomas/chroma/v2 v2.15.0/go.mod h1:gUhVLrPDXPtp/f+L1jo9xepo9gL4eL
github.com/alecthomas/repr v0.0.0-20220113201626-b1b626ac65ae/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8=
github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
+github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
@@ -107,6 +109,8 @@ github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm
github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA=
github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo=
+github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/golang/geo v0.0.0-20230421003525-6adc56603217 h1:HKlyj6in2JV6wVkmQ4XmG/EIm+SCYlPZ+V4GWit7Z+I=
@@ -168,8 +172,8 @@ github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0
github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU=
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
-github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
-github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
+github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
github.com/markbates/goth v1.80.0 h1:NnvatczZDzOs1hn9Ug+dVYf2Viwwkp/ZDX5K+GLjan8=
github.com/markbates/goth v1.80.0/go.mod h1:4/GYHo+W6NWisrMPZnq0Yr2Q70UntNLn7KXEFhrIdAY=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -180,6 +184,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/meilisearch/meilisearch-go v0.31.0 h1:yZRhY1qJqdH8h6GFZALGtkDLyj8f9v5aJpsNMyrUmnY=
+github.com/meilisearch/meilisearch-go v0.31.0/go.mod h1:aNtyuwurDg/ggxQIcKqWH6G9g2ptc8GyY7PLY4zMn/g=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -219,8 +225,13 @@ github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWR
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w=
@@ -233,6 +244,8 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
+github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
+github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
github.com/yuin/goldmark v1.4.15/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
github.com/yuin/goldmark v1.7.8 h1:iERMLn0/QJeHFhxSt3p6PeN9mGnvIKSpG9YYorDMnic=
diff --git a/internal/cli/main.go b/internal/cli/main.go
index f36f12b..5b07b89 100644
--- a/internal/cli/main.go
+++ b/internal/cli/main.go
@@ -124,9 +124,9 @@ func Initialize(ctx *cli.Context) {
log.Error().Err(err).Msg("Failed to initialize WebAuthn")
}
- if config.C.IndexEnabled {
- log.Info().Msg("Index directory: " + filepath.Join(homePath, config.C.IndexDirname))
- index.Init(filepath.Join(homePath, config.C.IndexDirname))
+ index.DepreactionIndexDirname()
+ if index.IndexEnabled() {
+ index.NewIndexer(index.IndexType())
}
}
@@ -136,7 +136,7 @@ func shutdown() {
log.Error().Err(err).Msg("Failed to close database")
}
- if config.C.IndexEnabled {
+ if index.IndexEnabled() {
log.Info().Msg("Shutting down index...")
index.Close()
}
diff --git a/internal/config/config.go b/internal/config/config.go
index cd62320..b1aa187 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -37,8 +37,11 @@ type config struct {
DBUri string `yaml:"db-uri" env:"OG_DB_URI"`
DBFilename string `yaml:"db-filename" env:"OG_DB_FILENAME"` // deprecated
- IndexEnabled bool `yaml:"index.enabled" env:"OG_INDEX_ENABLED"`
- IndexDirname string `yaml:"index.dirname" env:"OG_INDEX_DIRNAME"`
+ IndexEnabled bool `yaml:"index.enabled" env:"OG_INDEX_ENABLED"` // deprecated
+ Index string `yaml:"index" env:"OG_INDEX"`
+ BleveDirname string `yaml:"index.dirname" env:"OG_INDEX_DIRNAME"` // deprecated
+ MeiliHost string `yaml:"index.meili.host" env:"OG_MEILI_HOST"`
+ MeiliAPIKey string `yaml:"index.meili.api-key" env:"OG_MEILI_API_KEY"`
GitDefaultBranch string `yaml:"git.default-branch" env:"OG_GIT_DEFAULT_BRANCH"`
@@ -94,8 +97,7 @@ func configWithDefaults() (*config, error) {
c.LogOutput = "stdout,file"
c.OpengistHome = ""
c.DBUri = "opengist.db"
- c.IndexEnabled = true
- c.IndexDirname = "opengist.index"
+ c.Index = "bleve"
c.SqliteJournalMode = "WAL"
diff --git a/internal/db/gist.go b/internal/db/gist.go
index 0e9a475..1aca1ec 100644
--- a/internal/db/gist.go
+++ b/internal/db/gist.go
@@ -40,6 +40,10 @@ func (v Visibility) String() string {
}
}
+func (v Visibility) Uint() uint {
+ return uint(v)
+}
+
func (v Visibility) Next() Visibility {
switch v {
case PublicVisibility:
@@ -788,6 +792,8 @@ func (gist *Gist) ToIndexedGist() (*index.Gist, error) {
indexedGist := &index.Gist{
GistID: gist.ID,
+ UserID: gist.UserID,
+ Visibility: gist.Private.Uint(),
Username: gist.User.Username,
Title: gist.Title,
Content: wholeContent,
@@ -803,7 +809,7 @@ func (gist *Gist) ToIndexedGist() (*index.Gist, error) {
}
func (gist *Gist) AddInIndex() {
- if !index.Enabled() {
+ if !index.IndexEnabled() {
return
}
@@ -821,7 +827,7 @@ func (gist *Gist) AddInIndex() {
}
func (gist *Gist) RemoveFromIndex() {
- if !index.Enabled() {
+ if !index.IndexEnabled() {
return
}
diff --git a/internal/index/bleve.go b/internal/index/bleve.go
index 7212a65..8dec430 100644
--- a/internal/index/bleve.go
+++ b/internal/index/bleve.go
@@ -10,37 +10,32 @@ import (
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
"github.com/blevesearch/bleve/v2/search/query"
"github.com/rs/zerolog/log"
- "github.com/thomiceli/opengist/internal/config"
"strconv"
- "sync/atomic"
)
-var atomicIndexer atomic.Pointer[Indexer]
-
-type Indexer struct {
- Index bleve.Index
+type BleveIndexer struct {
+ index bleve.Index
+ path string
}
-func Enabled() bool {
- return config.C.IndexEnabled
+func NewBleveIndexer(path string) *BleveIndexer {
+ return &BleveIndexer{path: path}
}
-func Init(indexFilename string) {
- atomicIndexer.Store(&Indexer{Index: nil})
-
+func (i *BleveIndexer) Init() {
go func() {
- bleveIndex, err := open(indexFilename)
+ bleveIndex, err := i.open()
if err != nil {
- log.Error().Err(err).Msg("Failed to open index")
- (*atomicIndexer.Load()).close()
+ log.Error().Err(err).Msg("Failed to open Bleve index")
+ i.Close()
}
- atomicIndexer.Store(&Indexer{Index: bleveIndex})
- log.Info().Msg("Indexer initialized")
+ i.index = bleveIndex
+ log.Info().Msg("Bleve indexer initialized")
}()
}
-func open(indexFilename string) (bleve.Index, error) {
- bleveIndex, err := bleve.Open(indexFilename)
+func (i *BleveIndexer) open() (bleve.Index, error) {
+ bleveIndex, err := bleve.Open(i.path)
if err == nil {
return bleveIndex, nil
}
@@ -73,67 +68,33 @@ func open(indexFilename string) (bleve.Index, error) {
docMapping.DefaultAnalyzer = "gistAnalyser"
- return bleve.New(indexFilename, mapping)
+ return bleve.New(i.path, mapping)
}
-func Close() {
- (*atomicIndexer.Load()).close()
-}
-
-func (i *Indexer) close() {
- if i == nil || i.Index == nil {
+func (i *BleveIndexer) Close() {
+ if i == nil || i.index == nil {
return
}
- err := i.Index.Close()
+ err := i.index.Close()
if err != nil {
- log.Error().Err(err).Msg("Failed to close bleve index")
+ log.Error().Err(err).Msg("Failed to close Bleve index")
}
- log.Info().Msg("Indexer closed")
- atomicIndexer.Store(&Indexer{Index: nil})
+ log.Info().Msg("Bleve indexer closed")
}
-func checkForIndexer() error {
- if (*atomicIndexer.Load()).Index == nil {
- return errors.New("indexer is not initialized")
- }
-
- return nil
-}
-
-func AddInIndex(gist *Gist) error {
- if !Enabled() {
- return nil
- }
- if err := checkForIndexer(); err != nil {
- return err
- }
-
+func (i *BleveIndexer) Add(gist *Gist) error {
if gist == nil {
return errors.New("failed to add nil gist to index")
}
- return (*atomicIndexer.Load()).Index.Index(strconv.Itoa(int(gist.GistID)), gist)
+ return (*atomicIndexer.Load()).(*BleveIndexer).index.Index(strconv.Itoa(int(gist.GistID)), gist)
}
-func RemoveFromIndex(gistID uint) error {
- if !Enabled() {
- return nil
- }
- if err := checkForIndexer(); err != nil {
- return err
- }
-
- return (*atomicIndexer.Load()).Index.Delete(strconv.Itoa(int(gistID)))
+func (i *BleveIndexer) Remove(gistID uint) error {
+ return (*atomicIndexer.Load()).(*BleveIndexer).index.Delete(strconv.Itoa(int(gistID)))
}
-func SearchGists(queryStr string, queryMetadata SearchGistMetadata, gistsIds []uint, page int) ([]uint, uint64, map[string]int, error) {
- if !Enabled() {
- return nil, 0, nil, nil
- }
- if err := checkForIndexer(); err != nil {
- return nil, 0, nil, err
- }
-
+func (i *BleveIndexer) Search(queryStr string, queryMetadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
var err error
var indexerQuery query.Query
if queryStr != "" {
@@ -145,17 +106,16 @@ func SearchGists(queryStr string, queryMetadata SearchGistMetadata, gistsIds []u
indexerQuery = contentQuery
}
- repoQueries := make([]query.Query, 0, len(gistsIds))
+ privateQuery := bleve.NewBoolFieldQuery(false)
+ privateQuery.SetField("Private")
+ userIdMatch := float64(userId)
truee := true
- for _, id := range gistsIds {
- f := float64(id)
- qq := bleve.NewNumericRangeInclusiveQuery(&f, &f, &truee, &truee)
- qq.SetField("GistID")
- repoQueries = append(repoQueries, qq)
- }
+ userIdQuery := bleve.NewNumericRangeInclusiveQuery(&userIdMatch, &userIdMatch, &truee, &truee)
+ userIdQuery.SetField("UserID")
- indexerQuery = bleve.NewConjunctionQuery(bleve.NewDisjunctionQuery(repoQueries...), indexerQuery)
+ accessQuery := bleve.NewDisjunctionQuery(privateQuery, userIdQuery)
+ indexerQuery = bleve.NewConjunctionQuery(accessQuery, indexerQuery)
addQuery := func(field, value string) {
if value != "" && value != "." {
@@ -182,7 +142,7 @@ func SearchGists(queryStr string, queryMetadata SearchGistMetadata, gistsIds []u
s.Fields = []string{"GistID"}
s.IncludeLocations = false
- results, err := (*atomicIndexer.Load()).Index.Search(s)
+ results, err := (*atomicIndexer.Load()).(*BleveIndexer).index.Search(s)
if err != nil {
return nil, 0, nil, err
}
diff --git a/internal/index/gist.go b/internal/index/gist.go
index b9aa834..4db052e 100644
--- a/internal/index/gist.go
+++ b/internal/index/gist.go
@@ -2,6 +2,8 @@ package index
type Gist struct {
GistID uint
+ UserID uint
+ Visibility uint
Username string
Title string
Content string
diff --git a/internal/index/indexer.go b/internal/index/indexer.go
new file mode 100644
index 0000000..ea988e8
--- /dev/null
+++ b/internal/index/indexer.go
@@ -0,0 +1,136 @@
+package index
+
+import (
+ "fmt"
+ "github.com/rs/zerolog/log"
+ "github.com/thomiceli/opengist/internal/config"
+ "path/filepath"
+ "sync/atomic"
+)
+
+var atomicIndexer atomic.Pointer[Indexer]
+
+type Indexer interface {
+ Init()
+ Close()
+ Add(gist *Gist) error
+ Remove(gistID uint) error
+ Search(query string, metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error)
+}
+
+type IndexerType string
+
+const (
+ Bleve IndexerType = "bleve"
+ Meilisearch IndexerType = "meilisearch"
+ None IndexerType = ""
+)
+
+func IndexType() IndexerType {
+ switch config.C.Index {
+ case "bleve":
+ return Bleve
+ case "meilisearch":
+ return Meilisearch
+ default:
+ return None
+ }
+}
+
+func IndexEnabled() bool {
+ switch config.C.Index {
+ case "bleve", "meilisearch":
+ return true
+ default:
+ return false
+ }
+}
+
+func NewIndexer(idxType IndexerType) {
+ if !IndexEnabled() {
+ return
+ }
+ atomicIndexer.Store(nil)
+
+ var idx Indexer
+
+ switch idxType {
+ case Bleve:
+ idx = NewBleveIndexer(filepath.Join(config.GetHomeDir(), "opengist.index"))
+ case Meilisearch:
+ idx = NewMeiliIndexer(config.C.MeiliHost, config.C.MeiliAPIKey, "opengist")
+ default:
+ log.Warn().Msgf("Failed to create indexer, unknown indexer type: %s", idxType)
+ return
+ }
+
+ idx.Init()
+ atomicIndexer.Store(&idx)
+}
+
+func Close() {
+ if !IndexEnabled() {
+ return
+ }
+
+ idx := *atomicIndexer.Load()
+ if idx == nil {
+ return
+ }
+
+ idx.Close()
+ atomicIndexer.Store(nil)
+}
+
+func AddInIndex(gist *Gist) error {
+ if !IndexEnabled() {
+ return nil
+ }
+
+ idx := *atomicIndexer.Load()
+ if idx == nil {
+ return fmt.Errorf("indexer is not initialized")
+ }
+
+ return idx.Add(gist)
+}
+
+func RemoveFromIndex(gistID uint) error {
+ if !IndexEnabled() {
+ return nil
+ }
+
+ idx := *atomicIndexer.Load()
+ if idx == nil {
+ return fmt.Errorf("indexer is not initialized")
+ }
+
+ return idx.Remove(gistID)
+}
+
+func SearchGists(query string, metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
+ if !IndexEnabled() {
+ return nil, 0, nil, nil
+ }
+
+ idx := *atomicIndexer.Load()
+ if idx == nil {
+ return nil, 0, nil, fmt.Errorf("indexer is not initialized")
+ }
+
+ return idx.Search(query, metadata, userId, page)
+}
+
+func DepreactionIndexDirname() {
+ if config.C.IndexEnabled {
+ log.Warn().Msg("The 'index.enabled'/'OG_INDEX_ENABLED' configuration option is deprecated and will be removed in a future version. Please use 'index'/'OG_INDEX' instead.")
+ }
+
+ if config.C.Index == "" {
+ config.C.Index = "bleve"
+ }
+
+ if config.C.BleveDirname != "" {
+ log.Warn().Msg("The 'index.dirname'/'OG_INDEX_DIRNAME' configuration option is deprecated and will be removed in a future version.")
+ }
+}
diff --git a/internal/index/meilisearch.go b/internal/index/meilisearch.go
new file mode 100644
index 0000000..ad29258
--- /dev/null
+++ b/internal/index/meilisearch.go
@@ -0,0 +1,146 @@
+package index
+
+import (
+ "errors"
+ "fmt"
+ "github.com/meilisearch/meilisearch-go"
+ "github.com/rs/zerolog/log"
+ "strconv"
+ "strings"
+)
+
+type MeiliIndexer struct {
+ client meilisearch.ServiceManager
+ index meilisearch.IndexManager
+ indexName string
+ host string
+ apikey string
+}
+
+func NewMeiliIndexer(host, apikey, indexName string) *MeiliIndexer {
+ return &MeiliIndexer{
+ host: host,
+ apikey: apikey,
+ indexName: indexName,
+ }
+}
+
+func (i *MeiliIndexer) Init() {
+ go func() {
+ meiliIndex, err := i.open()
+ if err != nil {
+ log.Error().Err(err).Msg("Failed to open Meilisearch index")
+ i.Close()
+ }
+ i.index = meiliIndex
+ log.Info().Msg("Meilisearch indexer initialized")
+ }()
+}
+
+func (i *MeiliIndexer) open() (meilisearch.IndexManager, error) {
+ client := meilisearch.New(i.host, meilisearch.WithAPIKey(i.apikey))
+ indexResult, err := client.GetIndex(i.indexName)
+ if err != nil {
+ return nil, err
+ }
+
+ if indexResult != nil {
+ return indexResult.IndexManager, nil
+ }
+ _, err = client.CreateIndex(&meilisearch.IndexConfig{
+ Uid: i.indexName,
+ PrimaryKey: "GistID",
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ _, _ = client.Index(i.indexName).UpdateSettings(&meilisearch.Settings{
+ FilterableAttributes: []string{"GistID", "UserID", "Visibility", "Username", "Title", "Filenames", "Extensions", "Languages", "Topics"},
+ DisplayedAttributes: []string{"GistID"},
+ SearchableAttributes: []string{"Content", "Username", "Title", "Filenames", "Extensions", "Languages", "Topics"},
+ RankingRules: []string{"words"},
+ })
+
+ return client.Index(i.indexName), nil
+}
+
+func (i *MeiliIndexer) Close() {
+ if i.client != nil {
+ i.client.Close()
+ log.Info().Msg("Meilisearch indexer closed")
+ }
+ i.client = nil
+}
+
+func (i *MeiliIndexer) Add(gist *Gist) error {
+ if gist == nil {
+ return errors.New("failed to add nil gist to index")
+ }
+ _, err := (*atomicIndexer.Load()).(*MeiliIndexer).index.AddDocuments(gist, "GistID")
+ return err
+}
+
+func (i *MeiliIndexer) Remove(gistID uint) error {
+ _, err := (*atomicIndexer.Load()).(*MeiliIndexer).index.DeleteDocument(strconv.Itoa(int(gistID)))
+ return err
+}
+
+func (i *MeiliIndexer) Search(queryStr string, queryMetadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
+ searchRequest := &meilisearch.SearchRequest{
+ Offset: int64((page - 1) * 10),
+ Limit: 11,
+ AttributesToRetrieve: []string{"GistID", "Languages"},
+ Facets: []string{"Languages"},
+ AttributesToSearchOn: []string{"Content"},
+ }
+
+ var filters []string
+ filters = append(filters, fmt.Sprintf("(Visibility = 0 OR UserID = %d)", userId))
+
+ addFilter := func(field, value string) {
+ if value != "" && value != "." {
+ filters = append(filters, fmt.Sprintf("%s = \"%s\"", field, escapeFilterValue(value)))
+ }
+ }
+ addFilter("Username", queryMetadata.Username)
+ addFilter("Title", queryMetadata.Title)
+ addFilter("Filenames", queryMetadata.Filename)
+ addFilter("Extensions", queryMetadata.Extension)
+ addFilter("Languages", queryMetadata.Language)
+ addFilter("Topics", queryMetadata.Topic)
+
+ if len(filters) > 0 {
+ searchRequest.Filter = strings.Join(filters, " AND ")
+ }
+
+ response, err := (*atomicIndexer.Load()).(*MeiliIndexer).index.Search(queryStr, searchRequest)
+ if err != nil {
+ log.Error().Err(err).Msg("Failed to search Meilisearch index")
+ return nil, 0, nil, err
+ }
+
+ gistIds := make([]uint, 0, len(response.Hits))
+ for _, hit := range response.Hits {
+ if gistID, ok := hit.(map[string]interface{})["GistID"].(float64); ok {
+ gistIds = append(gistIds, uint(gistID))
+ }
+ }
+
+ languageCounts := make(map[string]int)
+ if facets, ok := response.FacetDistribution.(map[string]interface{})["Languages"]; ok {
+ for language, count := range facets.(map[string]interface{}) {
+ if countValue, ok := count.(float64); ok {
+ languageCounts[language] = int(countValue)
+ }
+ }
+ }
+
+ return gistIds, uint64(response.EstimatedTotalHits), languageCounts, nil
+}
+
+func escapeFilterValue(value string) string {
+ escaped := strings.ReplaceAll(value, "\\", "\\\\")
+ escaped = strings.ReplaceAll(escaped, "\"", "\\\"")
+ return escaped
+}
diff --git a/internal/web/handlers/gist/all.go b/internal/web/handlers/gist/all.go
index f811050..b6c29ea 100644
--- a/internal/web/handlers/gist/all.go
+++ b/internal/web/handlers/gist/all.go
@@ -179,12 +179,6 @@ func Search(ctx *context.Context) error {
currentUserId = 0
}
- var visibleGistsIds []uint
- visibleGistsIds, err = db.GetAllGistsVisibleByUser(currentUserId)
- if err != nil {
- return ctx.ErrorRes(500, "Error fetching gists", err)
- }
-
gistsIds, nbHits, langs, err := index.SearchGists(content, index.SearchGistMetadata{
Username: meta["user"],
Title: meta["title"],
@@ -192,7 +186,7 @@ func Search(ctx *context.Context) error {
Extension: meta["extension"],
Language: meta["language"],
Topic: meta["topic"],
- }, visibleGistsIds, pageInt)
+ }, currentUserId, pageInt)
if err != nil {
return ctx.ErrorRes(500, "Error searching gists", err)
}
diff --git a/internal/web/handlers/gist/edit.go b/internal/web/handlers/gist/edit.go
index d45c2d9..e914d84 100644
--- a/internal/web/handlers/gist/edit.go
+++ b/internal/web/handlers/gist/edit.go
@@ -70,6 +70,8 @@ func EditVisibility(ctx *context.Context) error {
return ctx.ErrorRes(500, "Error updating this gist", err)
}
+ gist.AddInIndex()
+
ctx.AddFlash(ctx.Tr("flash.gist.visibility-changed"), "success")
return ctx.RedirectTo("/" + gist.User.Username + "/" + gist.Identifier())
}
diff --git a/internal/web/server/renderer.go b/internal/web/server/renderer.go
index 8cd4905..3c8e4de 100644
--- a/internal/web/server/renderer.go
+++ b/internal/web/server/renderer.go
@@ -171,7 +171,7 @@ func (s *Server) setFuncMap() {
return strings.TrimSpace(resultBuilder.String())
},
- "indexEnabled": index.Enabled,
+ "indexEnabled": index.IndexEnabled,
"isUrl": func(s string) bool {
_, err := url.ParseRequestURI(s)
return err == nil
diff --git a/internal/web/server/router.go b/internal/web/server/router.go
index 88e270d..05267d2 100644
--- a/internal/web/server/router.go
+++ b/internal/web/server/router.go
@@ -98,7 +98,7 @@ func (s *Server) registerRoutes() {
r.GET("/all", gist.AllGists, checkRequireLogin, setAllGistsMode("all"))
- if index.Enabled() {
+ if index.IndexEnabled() {
r.GET("/search", gist.Search, checkRequireLogin)
} else {
r.GET("/search", gist.AllGists, checkRequireLogin, setAllGistsMode("search"))
diff --git a/internal/web/test/server.go b/internal/web/test/server.go
index 0cd7d2f..7625bf0 100644
--- a/internal/web/test/server.go
+++ b/internal/web/test/server.go
@@ -151,7 +151,7 @@ func Setup(t *testing.T) *TestServer {
git.ReposDirectory = filepath.Join("tests")
- config.C.IndexEnabled = false
+ config.C.Index = ""
config.C.LogLevel = "error"
config.InitLog()
diff --git a/templates/pages/admin_config.html b/templates/pages/admin_config.html
index 8276890..c4311f3 100644
--- a/templates/pages/admin_config.html
+++ b/templates/pages/admin_config.html
@@ -19,8 +19,9 @@
Opengist home{{ .c.OpengistHome }}
Database type{{ .dbtype }}{{ if eq .dbtype "SQLite" }} ({{ .c.SqliteJournalMode }}){{ end }}
Database name{{ .dbname }}
- Index Enabled{{ .c.IndexEnabled }}
- Index Dirname{{ .c.IndexDirname }}
+ Index{{ .c.Index }}
+ MeiliSearch Host{{ .c.MeiliHost }}
+ MeiliSearch API Key{{ if .c.MeiliAPIKey }}<defined>{{ end }}
Git default branch{{ .c.GitDefaultBranch }}