Problem: indexing search objects is slow
continuous-integration/drone/push Build is passing Details

Solution: batch indexing search objects
master
Peter Stuifzand 2 years ago
parent 19183da0f8
commit da11811e2d

@ -337,14 +337,19 @@ func (fp *FilePages) save(msg saveMessage) error {
if err != nil {
return fmt.Errorf("while creating search object %s: %w", page.Name, err)
}
batch := fp.index.NewBatch()
for _, so := range searchObjects {
if fp.index != nil {
err = fp.index.Index(so.ID, so)
err = batch.Index(so.ID, so)
if err != nil {
return fmt.Errorf("while indexing %s: %w", page.Name, err)
}
}
}
err = fp.index.Batch(batch)
if err != nil {
return fmt.Errorf("while indexing %s: %w", page.Name, err)
}
sw.Stop()
sw.Start("links")
err = saveLinksIncremental(fp.dirname, page.Title)

@ -37,7 +37,6 @@ import (
"strings"
"time"
"github.com/davecgh/go-spew/spew"
"p83.nl/go/ekster/pkg/util"
"p83.nl/go/indieauth"
"p83.nl/go/wiki/link"
@ -1208,13 +1207,25 @@ func main() {
err = repo.Save(id, block)
// update search index
sw := stopwatch{}
sw.Start("createSearchObjects")
searchObjects, err := createSearchObjects(id)
spew.Dump("searchObjects", searchObjects)
batch := searchIndex.NewBatch()
for _, so := range searchObjects {
searchIndex.Index(so.ID, so)
err = batch.Index(so.ID, so)
if err != nil {
log.Println(err)
}
}
err = searchIndex.Batch(batch)
if err != nil {
log.Println(err)
}
sw.Stop()
return
}))
http.HandleFunc("/links.json", func(w http.ResponseWriter, r *http.Request) {

Loading…
Cancel
Save