Problem: indexing search objects is slow

Solution: batch indexing search objects

block.go

@@ -25,23 +25,56 @@ import (
 )
 
 type BlockRepository interface {
-	Save(block Block) error
+	Save(id string, block Block) error
 	Load(id string) (Block, error)
 }
 
+type blockSaveMessage struct {
+	id    string
+	block Block
+}
+
 type blockRepo struct {
 	dirname string
+	saveC   chan blockSaveMessage
+	errC    chan error
 }
 
-func (br *blockRepo) Save(id string, block Block) error {
+func saveBlock(dirname, id string, block Block) error {
-	f, err := os.OpenFile(filepath.Join(br.dirname, BlocksDirectory, id), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+	f, err := os.OpenFile(filepath.Join(dirname, BlocksDirectory, id), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
-	defer f.Close()
 	if err != nil {
 		return err
 	}
+	defer f.Close()
 	enc := json.NewEncoder(f)
 	enc.SetIndent("", "  ")
-	return enc.Encode(&block)
+	err = enc.Encode(&block)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func NewBlockRepo(dirname string) BlockRepository {
+	saveC := make(chan blockSaveMessage, 1)
+	errC := make(chan error)
+	go func() {
+		for msg := range saveC {
+			err := saveBlock(dirname, msg.id, msg.block)
+			errC <- err
+		}
+	}()
+	return &blockRepo{
+		dirname: dirname,
+		saveC:   saveC,
+		errC:    errC,
+	}
+}
+
+func (br *blockRepo) Save(id string, block Block) error {
+	br.saveC <- blockSaveMessage{id, block}
+	err := <-br.errC
+	return err
 }
 
 func (br *blockRepo) Load(id string) (Block, error) {

file.go

@@ -337,14 +337,19 @@ func (fp *FilePages) save(msg saveMessage) error {
 		if err != nil {
 			return fmt.Errorf("while creating search object %s: %w", page.Name, err)
 		}
+		batch := fp.index.NewBatch()
 		for _, so := range searchObjects {
 			if fp.index != nil {
-				err = fp.index.Index(so.ID, so)
+				err = batch.Index(so.ID, so)
 				if err != nil {
 					return fmt.Errorf("while indexing %s: %w", page.Name, err)
 				}
 			}
 		}
+		err = fp.index.Batch(batch)
+		if err != nil {
+			return fmt.Errorf("while indexing %s: %w", page.Name, err)
+		}
 		sw.Stop()
 		sw.Start("links")
 		err = saveLinksIncremental(fp.dirname, page.Title)
@@ -699,6 +704,9 @@ func (fp *FilePages) PageHistory(p string) ([]Revision, error) {
 		commitId := line[0:start]
 		rest := line[start+1:]
 		pageText := gitRevision(fp.dirname, page, commitId)
+		if pageText == "" {
+			return nil, errors.New("git revision failed")
+		}
 		revisions = append(revisions, Revision{
 			Version: commitId,
 			Page:    DiffPage{Content: pageText},
@@ -727,8 +735,14 @@ func gitRevision(dirname, page, version string) string {
 	cmd.Dir = dirname
 	buf := bytes.Buffer{}
 	cmd.Stdout = &buf
-	cmd.Start()
+	err := cmd.Start()
-	cmd.Wait()
+	if err != nil {
+		return ""
+	}
+	err = cmd.Wait()
+	if err != nil {
+		return ""
+	}
 	return buf.String()
 }
 

list-editor/store.js

@@ -203,10 +203,10 @@ function Store(inputData) {
 
     function updateFold(id, open) {
         let closedFolds = JSON.parse(localStorage.getItem("closed-folds") || '{}') || {}
-        if (!open) {
+        if (open) {
-            closedFolds[id] = true
-        } else {
             delete closedFolds[id]
+        } else {
+            closedFolds[id] = true
         }
         localStorage.setItem('closed-folds', JSON.stringify(closedFolds))
     }

main.go

@@ -37,7 +37,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/davecgh/go-spew/spew"
 	"p83.nl/go/ekster/pkg/util"
 	"p83.nl/go/indieauth"
 	"p83.nl/go/wiki/link"
@@ -1064,8 +1063,39 @@ func main() {
 	}
 
 	mp = NewFilePages(dataDir, searchIndex)
+	repo := NewBlockRepo("data")
 
 	http.Handle("/auth/", &authHandler{})
+	http.HandleFunc("/api/block/view", wrapAuth(func(w http.ResponseWriter, r *http.Request) {
+		defer r.Body.Close()
+
+		if !r.Context().Value(authKey).(bool) {
+			http.Error(w, "Unauthorized", 401)
+			return
+		}
+
+		if r.Method != "GET" {
+			http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
+			return
+		}
+
+		id := r.URL.Query().Get("id")
+
+		block, err := repo.Load(id)
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+			return
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+		w.Header().Set("Cache-Control", "no-store")
+		enc := json.NewEncoder(w)
+		enc.SetIndent("", "  ")
+		err = enc.Encode(block)
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+		}
+	}))
 	http.HandleFunc("/api/block/", wrapAuth(func(w http.ResponseWriter, r *http.Request) {
 		defer r.Body.Close()
 
@@ -1134,7 +1164,6 @@ func main() {
 			http.Error(w, "missing id", 400)
 			return
 		}
-		repo := blockRepo{dirname: "data"}
 		block, err := repo.Load(id)
 		block.Text = r.Form.Get("text")
 		err = repo.Save(id, block)
@@ -1165,9 +1194,6 @@ func main() {
 			http.Error(w, "missing id", 400)
 			return
 		}
-		repo := blockRepo{
-			dirname: "data",
-		}
 		newBlock := Block{
 			Text:     r.Form.Get("text"),
 			Children: []string{},
@@ -1181,13 +1207,25 @@ func main() {
 		err = repo.Save(id, block)
 
 		// update search index
+		sw := stopwatch{}
+		sw.Start("createSearchObjects")
 		searchObjects, err := createSearchObjects(id)
-		spew.Dump("searchObjects", searchObjects)
+
+		batch := searchIndex.NewBatch()
 
 		for _, so := range searchObjects {
-			searchIndex.Index(so.ID, so)
+			err = batch.Index(so.ID, so)
+			if err != nil {
+				log.Println(err)
+			}
 		}
 
+		err = searchIndex.Batch(batch)
+		if err != nil {
+			log.Println(err)
+		}
+
+		sw.Stop()
 		return
 	}))
 	http.HandleFunc("/links.json", func(w http.ResponseWriter, r *http.Request) {