Problem: indexing search objects is slow
All checks were successful
continuous-integration/drone/push Build is passing
All checks were successful
continuous-integration/drone/push Build is passing
Solution: batch indexing search objects
This commit is contained in:
parent
19183da0f8
commit
da11811e2d
7
file.go
7
file.go
|
@ -337,14 +337,19 @@ func (fp *FilePages) save(msg saveMessage) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("while creating search object %s: %w", page.Name, err)
|
return fmt.Errorf("while creating search object %s: %w", page.Name, err)
|
||||||
}
|
}
|
||||||
|
batch := fp.index.NewBatch()
|
||||||
for _, so := range searchObjects {
|
for _, so := range searchObjects {
|
||||||
if fp.index != nil {
|
if fp.index != nil {
|
||||||
err = fp.index.Index(so.ID, so)
|
err = batch.Index(so.ID, so)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("while indexing %s: %w", page.Name, err)
|
return fmt.Errorf("while indexing %s: %w", page.Name, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
err = fp.index.Batch(batch)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("while indexing %s: %w", page.Name, err)
|
||||||
|
}
|
||||||
sw.Stop()
|
sw.Stop()
|
||||||
sw.Start("links")
|
sw.Start("links")
|
||||||
err = saveLinksIncremental(fp.dirname, page.Title)
|
err = saveLinksIncremental(fp.dirname, page.Title)
|
||||||
|
|
17
main.go
17
main.go
|
@ -37,7 +37,6 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/davecgh/go-spew/spew"
|
|
||||||
"p83.nl/go/ekster/pkg/util"
|
"p83.nl/go/ekster/pkg/util"
|
||||||
"p83.nl/go/indieauth"
|
"p83.nl/go/indieauth"
|
||||||
"p83.nl/go/wiki/link"
|
"p83.nl/go/wiki/link"
|
||||||
|
@ -1208,13 +1207,25 @@ func main() {
|
||||||
err = repo.Save(id, block)
|
err = repo.Save(id, block)
|
||||||
|
|
||||||
// update search index
|
// update search index
|
||||||
|
sw := stopwatch{}
|
||||||
|
sw.Start("createSearchObjects")
|
||||||
searchObjects, err := createSearchObjects(id)
|
searchObjects, err := createSearchObjects(id)
|
||||||
spew.Dump("searchObjects", searchObjects)
|
|
||||||
|
batch := searchIndex.NewBatch()
|
||||||
|
|
||||||
for _, so := range searchObjects {
|
for _, so := range searchObjects {
|
||||||
searchIndex.Index(so.ID, so)
|
err = batch.Index(so.ID, so)
|
||||||
|
if err != nil {
|
||||||
|
log.Println(err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
err = searchIndex.Batch(batch)
|
||||||
|
if err != nil {
|
||||||
|
log.Println(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sw.Stop()
|
||||||
return
|
return
|
||||||
}))
|
}))
|
||||||
http.HandleFunc("/links.json", func(w http.ResponseWriter, r *http.Request) {
|
http.HandleFunc("/links.json", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
Loading…
Reference in New Issue
Block a user