From 594f7ab96ce1d7cc9e335785289f7c654bcff72b Mon Sep 17 00:00:00 2001 From: Peter Stuifzand Date: Wed, 1 Jul 2020 16:40:10 +0200 Subject: [PATCH] Improve search --- backref.go | 15 +++++---- editor/src/index.js | 4 ++- file.go | 8 +++-- go.mod | 6 ++++ go.sum | 19 +++++++++++ main.go | 77 ++++++++++++++++++++++++++++----------------- search.go | 49 ++++++++++++++++++++++++++++- util.go | 2 +- 8 files changed, 140 insertions(+), 40 deletions(-) diff --git a/backref.go b/backref.go index 4fa97e3..0f18daa 100644 --- a/backref.go +++ b/backref.go @@ -14,6 +14,13 @@ type Reference struct { Name string } +// ListItem is a simplification of the information that was saved by the editor +type ListItem struct { + ID string + Indented int + Text string +} + type Refs map[string][]Reference func processBackrefs(dirname string, page Page) error { @@ -49,17 +56,13 @@ func saveBackrefs(filename string, refs Refs) error { func processBackrefsForPage(page Page, refs Refs) error { content := page.Content - var listItems []struct { - Id string - Indented int - Text string - } + var listItems []ListItem var links []ParsedLink err := json.NewDecoder(strings.NewReader(content)).Decode(&listItems) if err == nil { for _, item := range listItems { - foundLinks, err := ParseLinks(item.Id, item.Text) + foundLinks, err := ParseLinks(item.ID, item.Text) if err != nil { continue } diff --git a/editor/src/index.js b/editor/src/index.js index 9a6fb92..0c081d1 100644 --- a/editor/src/index.js +++ b/editor/src/index.js @@ -210,7 +210,9 @@ if (holder) { if (converted.startsWith("```", 0)) { converted = MD.render(converted) } else { - if (text.match(/#\[\[TODO]]/)) { + if (text.match(/^(\w+):: (.+)$/)) { + converted = converted.replace(/^(\w+):: (.*)$/, '**$1**: $2') + } else if (text.match(/#\[\[TODO]]/)) { converted = converted.replace('#[[TODO]]', '') } else if (text.match(/#\[\[DONE]]/)) { converted = converted.replace('#[[DONE]]', '') diff --git a/file.go b/file.go index 3d05000..fd2ccf9 100644 --- a/file.go +++ b/file.go @@ -130,7 +130,11 @@ func (fp *FilePages) save(msg saveMessage) error { } sw.Stop() sw.Start("index") - err = fp.index.Index(page.Name, page) + so, err := createSearchObject(page) + if err != nil { + return fmt.Errorf("while creating search object %s: %w", page.Name, err) + } + err = fp.index.Index(page.Name, so) if err != nil { return fmt.Errorf("while indexing %s: %w", page.Name, err) } @@ -452,7 +456,7 @@ func (fp *FilePages) AllPages() ([]Page, error) { var pages []Page for _, file := range files { - if file.Name()[0] == '.' { + if file.Name()[0] == '.' || file.Name()[0] == '_' { continue } if file.Name() == "backrefs.json" { diff --git a/go.mod b/go.mod index 8cab1f1..a2756a2 100644 --- a/go.mod +++ b/go.mod @@ -5,10 +5,16 @@ go 1.14 require ( github.com/RoaringBitmap/roaring v0.4.23 // indirect github.com/blevesearch/bleve v1.0.9 + github.com/blevesearch/cld2 v0.0.0-20200327141045-8b5f551d37f5 // indirect + github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d // indirect github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a // indirect github.com/golang/protobuf v1.4.2 // indirect + github.com/ikawaha/kagome.ipadic v1.1.2 // indirect + github.com/jmhodges/levigo v1.0.0 // indirect github.com/sergi/go-diff v1.1.0 github.com/stretchr/testify v1.4.0 + github.com/tebeka/snowball v0.4.2 // indirect + github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect github.com/tinylib/msgp v1.1.2 // indirect github.com/yuin/goldmark v1.1.32 gitlab.com/golang-commonmark/linkify v0.0.0-20200225224916-64bca66f6ad3 // indirect diff --git a/go.sum b/go.sum index 3655efc..061a47c 100644 --- a/go.sum +++ b/go.sum @@ -9,7 +9,10 @@ github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9Pq github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/blevesearch/bleve v1.0.9 h1:kqw/Ank/61UV9/Bx9kCcnfH6qWPgmS8O5LNfpsgzASg= github.com/blevesearch/bleve v1.0.9/go.mod h1:tb04/rbU29clbtNgorgFd8XdJea4x3ybYaOjWKr+UBU= +github.com/blevesearch/blevex v0.0.0-20190916190636-152f0fe5c040 h1:SjYVcfJVZoCfBlg+fkaq2eoZHTf5HaJfaTeTkOtyfHQ= github.com/blevesearch/blevex v0.0.0-20190916190636-152f0fe5c040/go.mod h1:WH+MU2F4T0VmSdaPX+Wu5GYoZBrYWdOZWSjzvYcDmqQ= +github.com/blevesearch/cld2 v0.0.0-20200327141045-8b5f551d37f5 h1:/4ikScMMYMqsRFWJjCyzd3CNWB0lxvqDkqa5nEv6NMc= +github.com/blevesearch/cld2 v0.0.0-20200327141045-8b5f551d37f5/go.mod h1:PN0QNTLs9+j1bKy3d/GB/59wsNBFC4sWLWG3k69lWbc= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= @@ -31,11 +34,15 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps= github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k= +github.com/couchbase/moss v0.1.0 h1:HCL+xxHUwmOaL44kMM/gU08OW6QGCui1WVFO58bjhNI= github.com/couchbase/moss v0.1.0/go.mod h1:9MaHIaRuy9pvLPUJxB8sh8OrLfyDczECVL37grCIubs= github.com/couchbase/vellum v1.0.1 h1:qrj9ohvZedvc51S5KzPfJ6P6z0Vqzv7Lx7k3mVc2WOk= github.com/couchbase/vellum v1.0.1/go.mod h1:FcwrEivFpNi24R3jLOs3n+fs5RnuQnQqCLBJ1uAg1W4= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d h1:SwD98825d6bdB+pEuTxWOXiSjBrHdOl/UVp75eI7JT8= +github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -71,7 +78,12 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ikawaha/kagome.ipadic v1.1.2 h1:pFxZ1PpMpc6ZoBK712YN5cVK0u/ju2DZ+gRIOriJFFs= +github.com/ikawaha/kagome.ipadic v1.1.2/go.mod h1:DPSBbU0czaJhAb/5uKQZHMc9MTVRpDugJfX+HddPHHg= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= +github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kljensen/snowball v0.6.0/go.mod h1:27N7E8fVU5H68RlUmnWwZCfxgt4POBJfENGMvNRhldw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -101,8 +113,10 @@ github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNX github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/steveyen/gtreap v0.1.0 h1:CjhzTa274PyJLJuMZwIzCO1PfC00oRa8d1Kc78bFXJM= @@ -111,7 +125,12 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/tebeka/snowball v0.4.2 h1:ujvgLOr6IHbsvB2Vgz27IcxWqDrNu9/oPhhe74lN/Kc= +github.com/tebeka/snowball v0.4.2/go.mod h1:4IfL14h1lvwZcp1sfXuuc7/7yCsvVffTWxWxCLfFpYg= +github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= +github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU= github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ= diff --git a/main.go b/main.go index 76e5314..93f1382 100644 --- a/main.go +++ b/main.go @@ -147,7 +147,7 @@ type recentPage struct { type indexHandler struct{} type graphHandler struct{} -type saveHandler struct{ +type saveHandler struct { SearchIndex bleve.Index } type editHandler struct{} @@ -341,11 +341,13 @@ func (h *saveHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { sess, err := NewSession(w, r) if err != nil { http.Error(w, err.Error(), 500) + log.Println(err) return } defer func() { if err := sess.Flush(); err != nil { log.Println(err) + log.Println(err) } }() @@ -357,6 +359,7 @@ func (h *saveHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { err = r.ParseForm() if err != nil { http.Error(w, err.Error(), 500) + log.Println(err) return } isJson := r.PostForm.Get("json") == "1" @@ -821,32 +824,10 @@ func main() { *baseurl = strings.TrimRight(*baseurl, "/") + "/" redirectURI = fmt.Sprintf("%sauth/callback", *baseurl) - indexFilename := "data/index.bleve" - - var searchIndex bleve.Index - if _, err := os.Stat(indexFilename); os.IsNotExist(err) { - indexMapping := bleve.NewIndexMapping() - searchIndex, err = bleve.New(indexFilename, indexMapping) - if err != nil { - log.Fatal(err) - } - - mp = NewFilePages("data", nil) - pages, err := mp.AllPages() - if err != nil { - log.Fatal(err) - } - for _, page:= range pages { - err = searchIndex.Index(page.Name, page) - if err != nil { - log.Println(err) - } - } - } else { - searchIndex, err = bleve.Open(indexFilename) - if err != nil { - log.Fatal(err) - } + dataDir := "data" + searchIndex, err := createSearchIndex(dataDir, "_page-index") + if err != nil { + log.Fatal(err) } defer searchIndex.Close() @@ -855,12 +836,12 @@ func main() { log.Fatal(err) } - mp = NewFilePages("data", searchIndex) + mp = NewFilePages(dataDir, searchIndex) http.Handle("/auth/", &authHandler{}) http.HandleFunc("/links.json", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - http.ServeFile(w, r, filepath.Join(mp.(*FilePages).dirname, LinksFile)) + http.ServeFile(w, r, filepath.Join(dataDir, LinksFile)) }) http.HandleFunc("/fetchLink", func(w http.ResponseWriter, r *http.Request) { link := r.URL.Query().Get("url") @@ -887,3 +868,41 @@ func main() { fmt.Printf("Running on port %d\n", *port) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) } + +func createSearchIndex(dataDir, indexName string) (bleve.Index, error) { + indexDir := filepath.Join(dataDir, indexName) + if _, err := os.Stat(indexDir); os.IsNotExist(err) { + indexMapping := bleve.NewIndexMapping() + searchIndex, err := bleve.New(indexDir, indexMapping) + if err != nil { + return nil, err + } + + fp := NewFilePages(dataDir, nil) + pages, err := fp.AllPages() + if err != nil { + return nil, err + } + + for _, page := range pages { + so, err := createSearchObject(page) + if err != nil { + log.Println(err) + continue + } + + err = searchIndex.Index(page.Name, so) + + if err != nil { + return nil, err + } + } + return searchIndex, nil + } else { + searchIndex, err := bleve.Open(indexDir) + if err != nil { + return nil, err + } + return searchIndex, nil + } +} diff --git a/search.go b/search.go index d1d1c0b..3a48074 100644 --- a/search.go +++ b/search.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "net/http" + "strings" "github.com/blevesearch/bleve" "github.com/blevesearch/bleve/mapping" @@ -19,9 +20,21 @@ type searchHandler struct { searchIndex bleve.Index } +type nameLine struct { + Name string `json:"name"` + Line string `json:"line"` +} + +type searchObject struct { + Title string `json:"title"` + Blocks []string `json:"blocks"` + Refs []nameLine `json:"refs"` + Meta map[string]string `json:"meta"` +} + func NewSearchHandler(searchIndex bleve.Index) (http.Handler, error) { return &searchHandler{ - searchIndex: searchIndex, + searchIndex: searchIndex, }, nil } @@ -39,3 +52,37 @@ func (s *searchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), 500) } } + +func createSearchObject(page Page) (searchObject, error) { + so := searchObject{} + so.Title = page.Title + so.Meta = make(map[string]string) + + type simpleListItem struct { + Text string + } + + var listItems []simpleListItem + if err := json.NewDecoder(strings.NewReader(page.Content)).Decode(&listItems); err != nil { + so.Blocks = append(so.Blocks, page.Content) + } else { + for _, li := range listItems { + meta := strings.SplitN(li.Text, "::", 2) + if len(meta) == 2 { + so.Meta[strings.ToLower(strings.TrimSpace(meta[0]))] = strings.ToLower(strings.TrimSpace(meta[1])) + } + so.Blocks = append(so.Blocks, li.Text) + } + } + + for k, refs := range page.Refs { + for _, ref := range refs { + so.Refs = append(so.Refs, nameLine{ + k, + ref.Line, + }) + } + } + + return so, nil +} diff --git a/util.go b/util.go index 309b409..393d564 100644 --- a/util.go +++ b/util.go @@ -10,7 +10,7 @@ import ( ) type ParsedLink struct { - ID string `json:"Id"` + ID string `json:"ID"` Name string PageName string Line string