Improve search
This commit is contained in:
parent
f18967729a
commit
594f7ab96c
15
backref.go
15
backref.go
|
|
@ -14,6 +14,13 @@ type Reference struct {
|
|||
Name string
|
||||
}
|
||||
|
||||
// ListItem is a simplification of the information that was saved by the editor
|
||||
type ListItem struct {
|
||||
ID string
|
||||
Indented int
|
||||
Text string
|
||||
}
|
||||
|
||||
type Refs map[string][]Reference
|
||||
|
||||
func processBackrefs(dirname string, page Page) error {
|
||||
|
|
@ -49,17 +56,13 @@ func saveBackrefs(filename string, refs Refs) error {
|
|||
func processBackrefsForPage(page Page, refs Refs) error {
|
||||
content := page.Content
|
||||
|
||||
var listItems []struct {
|
||||
Id string
|
||||
Indented int
|
||||
Text string
|
||||
}
|
||||
var listItems []ListItem
|
||||
|
||||
var links []ParsedLink
|
||||
err := json.NewDecoder(strings.NewReader(content)).Decode(&listItems)
|
||||
if err == nil {
|
||||
for _, item := range listItems {
|
||||
foundLinks, err := ParseLinks(item.Id, item.Text)
|
||||
foundLinks, err := ParseLinks(item.ID, item.Text)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
|
|
|||
|
|
@ -210,7 +210,9 @@ if (holder) {
|
|||
if (converted.startsWith("```", 0)) {
|
||||
converted = MD.render(converted)
|
||||
} else {
|
||||
if (text.match(/#\[\[TODO]]/)) {
|
||||
if (text.match(/^(\w+):: (.+)$/)) {
|
||||
converted = converted.replace(/^(\w+):: (.*)$/, '**$1**: $2')
|
||||
} else if (text.match(/#\[\[TODO]]/)) {
|
||||
converted = converted.replace('#[[TODO]]', '<input class="checkbox" type="checkbox" />')
|
||||
} else if (text.match(/#\[\[DONE]]/)) {
|
||||
converted = converted.replace('#[[DONE]]', '<input class="checkbox" type="checkbox" checked />')
|
||||
|
|
|
|||
8
file.go
8
file.go
|
|
@ -130,7 +130,11 @@ func (fp *FilePages) save(msg saveMessage) error {
|
|||
}
|
||||
sw.Stop()
|
||||
sw.Start("index")
|
||||
err = fp.index.Index(page.Name, page)
|
||||
so, err := createSearchObject(page)
|
||||
if err != nil {
|
||||
return fmt.Errorf("while creating search object %s: %w", page.Name, err)
|
||||
}
|
||||
err = fp.index.Index(page.Name, so)
|
||||
if err != nil {
|
||||
return fmt.Errorf("while indexing %s: %w", page.Name, err)
|
||||
}
|
||||
|
|
@ -452,7 +456,7 @@ func (fp *FilePages) AllPages() ([]Page, error) {
|
|||
|
||||
var pages []Page
|
||||
for _, file := range files {
|
||||
if file.Name()[0] == '.' {
|
||||
if file.Name()[0] == '.' || file.Name()[0] == '_' {
|
||||
continue
|
||||
}
|
||||
if file.Name() == "backrefs.json" {
|
||||
|
|
|
|||
6
go.mod
6
go.mod
|
|
@ -5,10 +5,16 @@ go 1.14
|
|||
require (
|
||||
github.com/RoaringBitmap/roaring v0.4.23 // indirect
|
||||
github.com/blevesearch/bleve v1.0.9
|
||||
github.com/blevesearch/cld2 v0.0.0-20200327141045-8b5f551d37f5 // indirect
|
||||
github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d // indirect
|
||||
github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a // indirect
|
||||
github.com/golang/protobuf v1.4.2 // indirect
|
||||
github.com/ikawaha/kagome.ipadic v1.1.2 // indirect
|
||||
github.com/jmhodges/levigo v1.0.0 // indirect
|
||||
github.com/sergi/go-diff v1.1.0
|
||||
github.com/stretchr/testify v1.4.0
|
||||
github.com/tebeka/snowball v0.4.2 // indirect
|
||||
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect
|
||||
github.com/tinylib/msgp v1.1.2 // indirect
|
||||
github.com/yuin/goldmark v1.1.32
|
||||
gitlab.com/golang-commonmark/linkify v0.0.0-20200225224916-64bca66f6ad3 // indirect
|
||||
|
|
|
|||
19
go.sum
19
go.sum
|
|
@ -9,7 +9,10 @@ github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9Pq
|
|||
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
|
||||
github.com/blevesearch/bleve v1.0.9 h1:kqw/Ank/61UV9/Bx9kCcnfH6qWPgmS8O5LNfpsgzASg=
|
||||
github.com/blevesearch/bleve v1.0.9/go.mod h1:tb04/rbU29clbtNgorgFd8XdJea4x3ybYaOjWKr+UBU=
|
||||
github.com/blevesearch/blevex v0.0.0-20190916190636-152f0fe5c040 h1:SjYVcfJVZoCfBlg+fkaq2eoZHTf5HaJfaTeTkOtyfHQ=
|
||||
github.com/blevesearch/blevex v0.0.0-20190916190636-152f0fe5c040/go.mod h1:WH+MU2F4T0VmSdaPX+Wu5GYoZBrYWdOZWSjzvYcDmqQ=
|
||||
github.com/blevesearch/cld2 v0.0.0-20200327141045-8b5f551d37f5 h1:/4ikScMMYMqsRFWJjCyzd3CNWB0lxvqDkqa5nEv6NMc=
|
||||
github.com/blevesearch/cld2 v0.0.0-20200327141045-8b5f551d37f5/go.mod h1:PN0QNTLs9+j1bKy3d/GB/59wsNBFC4sWLWG3k69lWbc=
|
||||
github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
|
||||
github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M=
|
||||
github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0=
|
||||
|
|
@ -31,11 +34,15 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
|
|||
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
|
||||
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps=
|
||||
github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k=
|
||||
github.com/couchbase/moss v0.1.0 h1:HCL+xxHUwmOaL44kMM/gU08OW6QGCui1WVFO58bjhNI=
|
||||
github.com/couchbase/moss v0.1.0/go.mod h1:9MaHIaRuy9pvLPUJxB8sh8OrLfyDczECVL37grCIubs=
|
||||
github.com/couchbase/vellum v1.0.1 h1:qrj9ohvZedvc51S5KzPfJ6P6z0Vqzv7Lx7k3mVc2WOk=
|
||||
github.com/couchbase/vellum v1.0.1/go.mod h1:FcwrEivFpNi24R3jLOs3n+fs5RnuQnQqCLBJ1uAg1W4=
|
||||
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
|
||||
github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d h1:SwD98825d6bdB+pEuTxWOXiSjBrHdOl/UVp75eI7JT8=
|
||||
github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
|
@ -71,7 +78,12 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
|
|||
github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/ikawaha/kagome.ipadic v1.1.2 h1:pFxZ1PpMpc6ZoBK712YN5cVK0u/ju2DZ+gRIOriJFFs=
|
||||
github.com/ikawaha/kagome.ipadic v1.1.2/go.mod h1:DPSBbU0czaJhAb/5uKQZHMc9MTVRpDugJfX+HddPHHg=
|
||||
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U=
|
||||
github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/kljensen/snowball v0.6.0/go.mod h1:27N7E8fVU5H68RlUmnWwZCfxgt4POBJfENGMvNRhldw=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
|
|
@ -101,8 +113,10 @@ github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNX
|
|||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
||||
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
||||
github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
|
||||
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
|
||||
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
|
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
|
||||
github.com/steveyen/gtreap v0.1.0 h1:CjhzTa274PyJLJuMZwIzCO1PfC00oRa8d1Kc78bFXJM=
|
||||
|
|
@ -111,7 +125,12 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
|
|||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
|
||||
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
|
||||
github.com/tebeka/snowball v0.4.2 h1:ujvgLOr6IHbsvB2Vgz27IcxWqDrNu9/oPhhe74lN/Kc=
|
||||
github.com/tebeka/snowball v0.4.2/go.mod h1:4IfL14h1lvwZcp1sfXuuc7/7yCsvVffTWxWxCLfFpYg=
|
||||
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok=
|
||||
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8=
|
||||
github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU=
|
||||
github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
|
||||
github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ=
|
||||
|
|
|
|||
77
main.go
77
main.go
|
|
@ -147,7 +147,7 @@ type recentPage struct {
|
|||
|
||||
type indexHandler struct{}
|
||||
type graphHandler struct{}
|
||||
type saveHandler struct{
|
||||
type saveHandler struct {
|
||||
SearchIndex bleve.Index
|
||||
}
|
||||
type editHandler struct{}
|
||||
|
|
@ -341,11 +341,13 @@ func (h *saveHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||
sess, err := NewSession(w, r)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 500)
|
||||
log.Println(err)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if err := sess.Flush(); err != nil {
|
||||
log.Println(err)
|
||||
log.Println(err)
|
||||
}
|
||||
}()
|
||||
|
||||
|
|
@ -357,6 +359,7 @@ func (h *saveHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||
err = r.ParseForm()
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 500)
|
||||
log.Println(err)
|
||||
return
|
||||
}
|
||||
isJson := r.PostForm.Get("json") == "1"
|
||||
|
|
@ -821,32 +824,10 @@ func main() {
|
|||
*baseurl = strings.TrimRight(*baseurl, "/") + "/"
|
||||
redirectURI = fmt.Sprintf("%sauth/callback", *baseurl)
|
||||
|
||||
indexFilename := "data/index.bleve"
|
||||
|
||||
var searchIndex bleve.Index
|
||||
if _, err := os.Stat(indexFilename); os.IsNotExist(err) {
|
||||
indexMapping := bleve.NewIndexMapping()
|
||||
searchIndex, err = bleve.New(indexFilename, indexMapping)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
mp = NewFilePages("data", nil)
|
||||
pages, err := mp.AllPages()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
for _, page:= range pages {
|
||||
err = searchIndex.Index(page.Name, page)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
searchIndex, err = bleve.Open(indexFilename)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
dataDir := "data"
|
||||
searchIndex, err := createSearchIndex(dataDir, "_page-index")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer searchIndex.Close()
|
||||
|
||||
|
|
@ -855,12 +836,12 @@ func main() {
|
|||
log.Fatal(err)
|
||||
}
|
||||
|
||||
mp = NewFilePages("data", searchIndex)
|
||||
mp = NewFilePages(dataDir, searchIndex)
|
||||
|
||||
http.Handle("/auth/", &authHandler{})
|
||||
http.HandleFunc("/links.json", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
http.ServeFile(w, r, filepath.Join(mp.(*FilePages).dirname, LinksFile))
|
||||
http.ServeFile(w, r, filepath.Join(dataDir, LinksFile))
|
||||
})
|
||||
http.HandleFunc("/fetchLink", func(w http.ResponseWriter, r *http.Request) {
|
||||
link := r.URL.Query().Get("url")
|
||||
|
|
@ -887,3 +868,41 @@ func main() {
|
|||
fmt.Printf("Running on port %d\n", *port)
|
||||
log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil))
|
||||
}
|
||||
|
||||
func createSearchIndex(dataDir, indexName string) (bleve.Index, error) {
|
||||
indexDir := filepath.Join(dataDir, indexName)
|
||||
if _, err := os.Stat(indexDir); os.IsNotExist(err) {
|
||||
indexMapping := bleve.NewIndexMapping()
|
||||
searchIndex, err := bleve.New(indexDir, indexMapping)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fp := NewFilePages(dataDir, nil)
|
||||
pages, err := fp.AllPages()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, page := range pages {
|
||||
so, err := createSearchObject(page)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
continue
|
||||
}
|
||||
|
||||
err = searchIndex.Index(page.Name, so)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return searchIndex, nil
|
||||
} else {
|
||||
searchIndex, err := bleve.Open(indexDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return searchIndex, nil
|
||||
}
|
||||
}
|
||||
|
|
|
|||
49
search.go
49
search.go
|
|
@ -3,6 +3,7 @@ package main
|
|||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/blevesearch/bleve"
|
||||
"github.com/blevesearch/bleve/mapping"
|
||||
|
|
@ -19,9 +20,21 @@ type searchHandler struct {
|
|||
searchIndex bleve.Index
|
||||
}
|
||||
|
||||
type nameLine struct {
|
||||
Name string `json:"name"`
|
||||
Line string `json:"line"`
|
||||
}
|
||||
|
||||
type searchObject struct {
|
||||
Title string `json:"title"`
|
||||
Blocks []string `json:"blocks"`
|
||||
Refs []nameLine `json:"refs"`
|
||||
Meta map[string]string `json:"meta"`
|
||||
}
|
||||
|
||||
func NewSearchHandler(searchIndex bleve.Index) (http.Handler, error) {
|
||||
return &searchHandler{
|
||||
searchIndex: searchIndex,
|
||||
searchIndex: searchIndex,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
|
@ -39,3 +52,37 @@ func (s *searchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||
http.Error(w, err.Error(), 500)
|
||||
}
|
||||
}
|
||||
|
||||
func createSearchObject(page Page) (searchObject, error) {
|
||||
so := searchObject{}
|
||||
so.Title = page.Title
|
||||
so.Meta = make(map[string]string)
|
||||
|
||||
type simpleListItem struct {
|
||||
Text string
|
||||
}
|
||||
|
||||
var listItems []simpleListItem
|
||||
if err := json.NewDecoder(strings.NewReader(page.Content)).Decode(&listItems); err != nil {
|
||||
so.Blocks = append(so.Blocks, page.Content)
|
||||
} else {
|
||||
for _, li := range listItems {
|
||||
meta := strings.SplitN(li.Text, "::", 2)
|
||||
if len(meta) == 2 {
|
||||
so.Meta[strings.ToLower(strings.TrimSpace(meta[0]))] = strings.ToLower(strings.TrimSpace(meta[1]))
|
||||
}
|
||||
so.Blocks = append(so.Blocks, li.Text)
|
||||
}
|
||||
}
|
||||
|
||||
for k, refs := range page.Refs {
|
||||
for _, ref := range refs {
|
||||
so.Refs = append(so.Refs, nameLine{
|
||||
k,
|
||||
ref.Line,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return so, nil
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user