diff --git a/backref.go b/backref.go
index 4fa97e3..0f18daa 100644
--- a/backref.go
+++ b/backref.go
@@ -14,6 +14,13 @@ type Reference struct {
Name string
}
+// ListItem is a simplification of the information that was saved by the editor
+type ListItem struct {
+ ID string
+ Indented int
+ Text string
+}
+
type Refs map[string][]Reference
func processBackrefs(dirname string, page Page) error {
@@ -49,17 +56,13 @@ func saveBackrefs(filename string, refs Refs) error {
func processBackrefsForPage(page Page, refs Refs) error {
content := page.Content
- var listItems []struct {
- Id string
- Indented int
- Text string
- }
+ var listItems []ListItem
var links []ParsedLink
err := json.NewDecoder(strings.NewReader(content)).Decode(&listItems)
if err == nil {
for _, item := range listItems {
- foundLinks, err := ParseLinks(item.Id, item.Text)
+ foundLinks, err := ParseLinks(item.ID, item.Text)
if err != nil {
continue
}
diff --git a/editor/src/index.js b/editor/src/index.js
index 9a6fb92..0c081d1 100644
--- a/editor/src/index.js
+++ b/editor/src/index.js
@@ -210,7 +210,9 @@ if (holder) {
if (converted.startsWith("```", 0)) {
converted = MD.render(converted)
} else {
- if (text.match(/#\[\[TODO]]/)) {
+ if (text.match(/^(\w+):: (.+)$/)) {
+ converted = converted.replace(/^(\w+):: (.*)$/, '**$1**: $2')
+ } else if (text.match(/#\[\[TODO]]/)) {
converted = converted.replace('#[[TODO]]', '')
} else if (text.match(/#\[\[DONE]]/)) {
converted = converted.replace('#[[DONE]]', '')
diff --git a/file.go b/file.go
index 3d05000..fd2ccf9 100644
--- a/file.go
+++ b/file.go
@@ -130,7 +130,11 @@ func (fp *FilePages) save(msg saveMessage) error {
}
sw.Stop()
sw.Start("index")
- err = fp.index.Index(page.Name, page)
+ so, err := createSearchObject(page)
+ if err != nil {
+ return fmt.Errorf("while creating search object %s: %w", page.Name, err)
+ }
+ err = fp.index.Index(page.Name, so)
if err != nil {
return fmt.Errorf("while indexing %s: %w", page.Name, err)
}
@@ -452,7 +456,7 @@ func (fp *FilePages) AllPages() ([]Page, error) {
var pages []Page
for _, file := range files {
- if file.Name()[0] == '.' {
+ if file.Name()[0] == '.' || file.Name()[0] == '_' {
continue
}
if file.Name() == "backrefs.json" {
diff --git a/go.mod b/go.mod
index 8cab1f1..a2756a2 100644
--- a/go.mod
+++ b/go.mod
@@ -5,10 +5,16 @@ go 1.14
require (
github.com/RoaringBitmap/roaring v0.4.23 // indirect
github.com/blevesearch/bleve v1.0.9
+ github.com/blevesearch/cld2 v0.0.0-20200327141045-8b5f551d37f5 // indirect
+ github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d // indirect
github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a // indirect
github.com/golang/protobuf v1.4.2 // indirect
+ github.com/ikawaha/kagome.ipadic v1.1.2 // indirect
+ github.com/jmhodges/levigo v1.0.0 // indirect
github.com/sergi/go-diff v1.1.0
github.com/stretchr/testify v1.4.0
+ github.com/tebeka/snowball v0.4.2 // indirect
+ github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect
github.com/tinylib/msgp v1.1.2 // indirect
github.com/yuin/goldmark v1.1.32
gitlab.com/golang-commonmark/linkify v0.0.0-20200225224916-64bca66f6ad3 // indirect
diff --git a/go.sum b/go.sum
index 3655efc..061a47c 100644
--- a/go.sum
+++ b/go.sum
@@ -9,7 +9,10 @@ github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9Pq
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/blevesearch/bleve v1.0.9 h1:kqw/Ank/61UV9/Bx9kCcnfH6qWPgmS8O5LNfpsgzASg=
github.com/blevesearch/bleve v1.0.9/go.mod h1:tb04/rbU29clbtNgorgFd8XdJea4x3ybYaOjWKr+UBU=
+github.com/blevesearch/blevex v0.0.0-20190916190636-152f0fe5c040 h1:SjYVcfJVZoCfBlg+fkaq2eoZHTf5HaJfaTeTkOtyfHQ=
github.com/blevesearch/blevex v0.0.0-20190916190636-152f0fe5c040/go.mod h1:WH+MU2F4T0VmSdaPX+Wu5GYoZBrYWdOZWSjzvYcDmqQ=
+github.com/blevesearch/cld2 v0.0.0-20200327141045-8b5f551d37f5 h1:/4ikScMMYMqsRFWJjCyzd3CNWB0lxvqDkqa5nEv6NMc=
+github.com/blevesearch/cld2 v0.0.0-20200327141045-8b5f551d37f5/go.mod h1:PN0QNTLs9+j1bKy3d/GB/59wsNBFC4sWLWG3k69lWbc=
github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M=
github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0=
@@ -31,11 +34,15 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps=
github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k=
+github.com/couchbase/moss v0.1.0 h1:HCL+xxHUwmOaL44kMM/gU08OW6QGCui1WVFO58bjhNI=
github.com/couchbase/moss v0.1.0/go.mod h1:9MaHIaRuy9pvLPUJxB8sh8OrLfyDczECVL37grCIubs=
github.com/couchbase/vellum v1.0.1 h1:qrj9ohvZedvc51S5KzPfJ6P6z0Vqzv7Lx7k3mVc2WOk=
github.com/couchbase/vellum v1.0.1/go.mod h1:FcwrEivFpNi24R3jLOs3n+fs5RnuQnQqCLBJ1uAg1W4=
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d h1:SwD98825d6bdB+pEuTxWOXiSjBrHdOl/UVp75eI7JT8=
+github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -71,7 +78,12 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/ikawaha/kagome.ipadic v1.1.2 h1:pFxZ1PpMpc6ZoBK712YN5cVK0u/ju2DZ+gRIOriJFFs=
+github.com/ikawaha/kagome.ipadic v1.1.2/go.mod h1:DPSBbU0czaJhAb/5uKQZHMc9MTVRpDugJfX+HddPHHg=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U=
+github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/kljensen/snowball v0.6.0/go.mod h1:27N7E8fVU5H68RlUmnWwZCfxgt4POBJfENGMvNRhldw=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -101,8 +113,10 @@ github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNX
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/steveyen/gtreap v0.1.0 h1:CjhzTa274PyJLJuMZwIzCO1PfC00oRa8d1Kc78bFXJM=
@@ -111,7 +125,12 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
+github.com/tebeka/snowball v0.4.2 h1:ujvgLOr6IHbsvB2Vgz27IcxWqDrNu9/oPhhe74lN/Kc=
+github.com/tebeka/snowball v0.4.2/go.mod h1:4IfL14h1lvwZcp1sfXuuc7/7yCsvVffTWxWxCLfFpYg=
+github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok=
+github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8=
github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU=
github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ=
diff --git a/main.go b/main.go
index 76e5314..93f1382 100644
--- a/main.go
+++ b/main.go
@@ -147,7 +147,7 @@ type recentPage struct {
type indexHandler struct{}
type graphHandler struct{}
-type saveHandler struct{
+type saveHandler struct {
SearchIndex bleve.Index
}
type editHandler struct{}
@@ -341,11 +341,13 @@ func (h *saveHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
sess, err := NewSession(w, r)
if err != nil {
http.Error(w, err.Error(), 500)
+ log.Println(err)
return
}
defer func() {
if err := sess.Flush(); err != nil {
log.Println(err)
+ log.Println(err)
}
}()
@@ -357,6 +359,7 @@ func (h *saveHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
err = r.ParseForm()
if err != nil {
http.Error(w, err.Error(), 500)
+ log.Println(err)
return
}
isJson := r.PostForm.Get("json") == "1"
@@ -821,32 +824,10 @@ func main() {
*baseurl = strings.TrimRight(*baseurl, "/") + "/"
redirectURI = fmt.Sprintf("%sauth/callback", *baseurl)
- indexFilename := "data/index.bleve"
-
- var searchIndex bleve.Index
- if _, err := os.Stat(indexFilename); os.IsNotExist(err) {
- indexMapping := bleve.NewIndexMapping()
- searchIndex, err = bleve.New(indexFilename, indexMapping)
- if err != nil {
- log.Fatal(err)
- }
-
- mp = NewFilePages("data", nil)
- pages, err := mp.AllPages()
- if err != nil {
- log.Fatal(err)
- }
- for _, page:= range pages {
- err = searchIndex.Index(page.Name, page)
- if err != nil {
- log.Println(err)
- }
- }
- } else {
- searchIndex, err = bleve.Open(indexFilename)
- if err != nil {
- log.Fatal(err)
- }
+ dataDir := "data"
+ searchIndex, err := createSearchIndex(dataDir, "_page-index")
+ if err != nil {
+ log.Fatal(err)
}
defer searchIndex.Close()
@@ -855,12 +836,12 @@ func main() {
log.Fatal(err)
}
- mp = NewFilePages("data", searchIndex)
+ mp = NewFilePages(dataDir, searchIndex)
http.Handle("/auth/", &authHandler{})
http.HandleFunc("/links.json", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
- http.ServeFile(w, r, filepath.Join(mp.(*FilePages).dirname, LinksFile))
+ http.ServeFile(w, r, filepath.Join(dataDir, LinksFile))
})
http.HandleFunc("/fetchLink", func(w http.ResponseWriter, r *http.Request) {
link := r.URL.Query().Get("url")
@@ -887,3 +868,41 @@ func main() {
fmt.Printf("Running on port %d\n", *port)
log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil))
}
+
+func createSearchIndex(dataDir, indexName string) (bleve.Index, error) {
+ indexDir := filepath.Join(dataDir, indexName)
+ if _, err := os.Stat(indexDir); os.IsNotExist(err) {
+ indexMapping := bleve.NewIndexMapping()
+ searchIndex, err := bleve.New(indexDir, indexMapping)
+ if err != nil {
+ return nil, err
+ }
+
+ fp := NewFilePages(dataDir, nil)
+ pages, err := fp.AllPages()
+ if err != nil {
+ return nil, err
+ }
+
+ for _, page := range pages {
+ so, err := createSearchObject(page)
+ if err != nil {
+ log.Println(err)
+ continue
+ }
+
+ err = searchIndex.Index(page.Name, so)
+
+ if err != nil {
+ return nil, err
+ }
+ }
+ return searchIndex, nil
+ } else {
+ searchIndex, err := bleve.Open(indexDir)
+ if err != nil {
+ return nil, err
+ }
+ return searchIndex, nil
+ }
+}
diff --git a/search.go b/search.go
index d1d1c0b..3a48074 100644
--- a/search.go
+++ b/search.go
@@ -3,6 +3,7 @@ package main
import (
"encoding/json"
"net/http"
+ "strings"
"github.com/blevesearch/bleve"
"github.com/blevesearch/bleve/mapping"
@@ -19,9 +20,21 @@ type searchHandler struct {
searchIndex bleve.Index
}
+type nameLine struct {
+ Name string `json:"name"`
+ Line string `json:"line"`
+}
+
+type searchObject struct {
+ Title string `json:"title"`
+ Blocks []string `json:"blocks"`
+ Refs []nameLine `json:"refs"`
+ Meta map[string]string `json:"meta"`
+}
+
func NewSearchHandler(searchIndex bleve.Index) (http.Handler, error) {
return &searchHandler{
- searchIndex: searchIndex,
+ searchIndex: searchIndex,
}, nil
}
@@ -39,3 +52,37 @@ func (s *searchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
http.Error(w, err.Error(), 500)
}
}
+
+func createSearchObject(page Page) (searchObject, error) {
+ so := searchObject{}
+ so.Title = page.Title
+ so.Meta = make(map[string]string)
+
+ type simpleListItem struct {
+ Text string
+ }
+
+ var listItems []simpleListItem
+ if err := json.NewDecoder(strings.NewReader(page.Content)).Decode(&listItems); err != nil {
+ so.Blocks = append(so.Blocks, page.Content)
+ } else {
+ for _, li := range listItems {
+ meta := strings.SplitN(li.Text, "::", 2)
+ if len(meta) == 2 {
+ so.Meta[strings.ToLower(strings.TrimSpace(meta[0]))] = strings.ToLower(strings.TrimSpace(meta[1]))
+ }
+ so.Blocks = append(so.Blocks, li.Text)
+ }
+ }
+
+ for k, refs := range page.Refs {
+ for _, ref := range refs {
+ so.Refs = append(so.Refs, nameLine{
+ k,
+ ref.Line,
+ })
+ }
+ }
+
+ return so, nil
+}
diff --git a/util.go b/util.go
index 309b409..393d564 100644
--- a/util.go
+++ b/util.go
@@ -10,7 +10,7 @@ import (
)
type ParsedLink struct {
- ID string `json:"Id"`
+ ID string `json:"ID"`
Name string
PageName string
Line string