Problem: indexing search objects is slow
All checks were successful
continuous-integration/drone/push Build is passing

Solution: batch indexing search objects
This commit is contained in:
Peter Stuifzand 2022-05-26 21:18:19 +02:00
parent 19183da0f8
commit da11811e2d
2 changed files with 20 additions and 4 deletions

View File

@ -337,14 +337,19 @@ func (fp *FilePages) save(msg saveMessage) error {
if err != nil { if err != nil {
return fmt.Errorf("while creating search object %s: %w", page.Name, err) return fmt.Errorf("while creating search object %s: %w", page.Name, err)
} }
batch := fp.index.NewBatch()
for _, so := range searchObjects { for _, so := range searchObjects {
if fp.index != nil { if fp.index != nil {
err = fp.index.Index(so.ID, so) err = batch.Index(so.ID, so)
if err != nil { if err != nil {
return fmt.Errorf("while indexing %s: %w", page.Name, err) return fmt.Errorf("while indexing %s: %w", page.Name, err)
} }
} }
} }
err = fp.index.Batch(batch)
if err != nil {
return fmt.Errorf("while indexing %s: %w", page.Name, err)
}
sw.Stop() sw.Stop()
sw.Start("links") sw.Start("links")
err = saveLinksIncremental(fp.dirname, page.Title) err = saveLinksIncremental(fp.dirname, page.Title)

17
main.go
View File

@ -37,7 +37,6 @@ import (
"strings" "strings"
"time" "time"
"github.com/davecgh/go-spew/spew"
"p83.nl/go/ekster/pkg/util" "p83.nl/go/ekster/pkg/util"
"p83.nl/go/indieauth" "p83.nl/go/indieauth"
"p83.nl/go/wiki/link" "p83.nl/go/wiki/link"
@ -1208,13 +1207,25 @@ func main() {
err = repo.Save(id, block) err = repo.Save(id, block)
// update search index // update search index
sw := stopwatch{}
sw.Start("createSearchObjects")
searchObjects, err := createSearchObjects(id) searchObjects, err := createSearchObjects(id)
spew.Dump("searchObjects", searchObjects)
batch := searchIndex.NewBatch()
for _, so := range searchObjects { for _, so := range searchObjects {
searchIndex.Index(so.ID, so) err = batch.Index(so.ID, so)
if err != nil {
log.Println(err)
}
} }
err = searchIndex.Batch(batch)
if err != nil {
log.Println(err)
}
sw.Stop()
return return
})) }))
http.HandleFunc("/links.json", func(w http.ResponseWriter, r *http.Request) { http.HandleFunc("/links.json", func(w http.ResponseWriter, r *http.Request) {