From 62bad5ec9148f915565a691debad11cc0f987dd9 Mon Sep 17 00:00:00 2001 From: Peter Stuifzand Date: Tue, 30 Jun 2020 22:56:12 +0200 Subject: [PATCH] Use bleve search and improve save times --- backref.go | 109 ++++++++++++--------- editor/src/fuse.js | 12 ++- editor/src/index.js | 75 +++++++-------- editor/src/search.js | 48 +++------- file.go | 82 ++++++++++++---- go.mod | 24 +++++ go.sum | 224 +++++++++++++++++++++++++++++++++++++++++++ main.go | 46 +++++++-- search.go | 41 ++++++++ util.go | 17 ++++ 10 files changed, 536 insertions(+), 142 deletions(-) create mode 100644 go.mod create mode 100644 go.sum create mode 100644 search.go diff --git a/backref.go b/backref.go index 0c5ae0f..b316af2 100644 --- a/backref.go +++ b/backref.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "fmt" "html/template" "os" "path/filepath" @@ -15,67 +16,74 @@ type Reference struct { type Refs map[string][]Reference -func processBackrefs(fp *FilePages) error { - pages, err := fp.AllPages() - +func processBackrefs(dirname string, page Page) error { + filename := filepath.Join(dirname, "backrefs.json") refs := make(Refs) + err := loadBackrefs(filename, refs) if err != nil { - return err + return fmt.Errorf("while loading backrefs: %w", err) } - for _, page := range pages { - content := page.Content - - var listItems []struct { - Indented int - Text string - } - - err = json.NewDecoder(strings.NewReader(content)).Decode(&listItems) - if err == nil { - pageText := "" - for _, item := range listItems { - pageText += strings.Repeat(" ", item.Indented) + "* " + item.Text + "\n" - } - - content = pageText - } - - links, err := ParseLinks(content) - - if err != nil { - return err - } - - for _, link := range links { - refs[link.PageName] = append(refs[link.PageName], Reference{link, page.Name}) - } + err = processBackrefsForPage(page, refs) + if err != nil { + return fmt.Errorf("while processing backrefs for %q: %w", page.Name, err) } - f, err := os.Create(filepath.Join(fp.dirname, "backrefs.json")) + if err = saveBackrefs(filename, refs); err != nil { + return fmt.Errorf("while saving backrefs: %w", err) + } + return nil +} + +func saveBackrefs(filename string, refs Refs) error { + f, err := os.Create(filename) if err != nil { return err } defer f.Close() - err = json.NewEncoder(f).Encode(&refs) - - return err + return json.NewEncoder(f).Encode(&refs) } -func loadBackrefs(fp *FilePages, p string) (map[string][]Backref, error) { +func processBackrefsForPage(page Page, refs Refs) error { + content := page.Content + + var listItems []struct { + Indented int + Text string + } + + err := json.NewDecoder(strings.NewReader(content)).Decode(&listItems) + if err == nil { + pageText := "" + for _, item := range listItems { + pageText += strings.Repeat(" ", item.Indented) + "* " + item.Text + "\n" + } + + content = pageText + } + + links, err := ParseLinks(content) + + if err != nil { + return err + } + + for _, link := range links { + // FIXME: this produces duplicates + refs[link.PageName] = append(refs[link.PageName], Reference{link, page.Name}) + } + + return nil +} + +func getBackrefs(fp *FilePages, p string) (map[string][]Backref, error) { refs := make(Refs) p = strings.Replace(p, " ", "_", -1) - f, err := os.Open(filepath.Join(fp.dirname, "backrefs.json")) - if err != nil { - return nil, err - } - - defer f.Close() - - err = json.NewDecoder(f).Decode(&refs) + filename := filepath.Join(fp.dirname, "backrefs.json") + err := loadBackrefs(filename, refs) if err != nil { return nil, err } @@ -112,3 +120,16 @@ func loadBackrefs(fp *FilePages, p string) (map[string][]Backref, error) { return result, nil } + +func loadBackrefs(filename string, refs Refs) error { + f, err := os.Open(filename) + if err != nil { + return err + } + defer f.Close() + err = json.NewDecoder(f).Decode(&refs) + if err != nil { + return err + } + return nil +} diff --git a/editor/src/fuse.js b/editor/src/fuse.js index 58602fc..3ef2f47 100644 --- a/editor/src/fuse.js +++ b/editor/src/fuse.js @@ -26,8 +26,16 @@ function createTitleSearch() { resolve({ documents, - titleSearch: titleFuse, - commandSearch: commandFuse, + titleSearch: query => { + return new Promise((resolve, reject) => { + resolve(titleFuse.search(query)) + }) + }, + commandSearch: query => { + return new Promise((resolve, reject) => { + resolve(commandFuse.search(query)) + }) + }, commands: commands, }) }) diff --git a/editor/src/index.js b/editor/src/index.js index ea77bd9..9a6fb92 100644 --- a/editor/src/index.js +++ b/editor/src/index.js @@ -86,7 +86,7 @@ function addIndicator(editor, indicator) { let holder = document.getElementById('editor'); function showSearchResults(searchTool, query, input, value, resultType) { - showSearchResultsExtended('#link-complete', 'link-template', searchTool, query, input, value, resultType, { + return showSearchResultsExtended('#link-complete', 'link-template', searchTool, query, input, value, resultType, { showOnlyResults: true, belowCursor: true }) @@ -94,38 +94,40 @@ function showSearchResults(searchTool, query, input, value, resultType) { function showSearchResultsExtended(element, template, searchTool, query, input, value, resultType, options) { const $lc = $(element) - let results = searchTool(query) - - let opt = options || {}; - - if (opt.showOnlyResults && (query.length === 0 || !results.length)) { - $lc.fadeOut() - return - } - - $lc.data('result-type', resultType) - - if (opt.belowCursor) { - let pos = getCaretCoordinates(input, value.selectionEnd, {}) - let off = $(input).offset() - pos.top += off.top + pos.height - pos.left += off.left - $lc.offset(pos) - } - - var templateText = document.getElementById(template).innerHTML; - var rendered = Mustache.render(templateText, {page: value.trim().replace(/\s+/g, '_'), results: results}, {}, ['[[', ']]']); - let selected = $lc.find('li.selected'); - if (selected) { - let selectedPos = $lc.find('li').index(selected[0]) - rendered = $(rendered) - const $lis = $lc.find('li') - if ($lis.length >= 1) { - selectedPos = Math.min(selectedPos, $lis.length - 1) - rendered.find('li')[selectedPos].classList.add('selected') + return searchTool(query).then(results => { + let opt = options || {}; + if (opt.showOnlyResults && (query.length === 0 || !results.length)) { + $lc.fadeOut() + return } - } - $lc.html(rendered).fadeIn() + $lc.data('result-type', resultType) + + if (opt.belowCursor) { + let pos = getCaretCoordinates(input, value.selectionEnd, {}) + let off = $(input).offset() + pos.top += off.top + pos.height + pos.left += off.left + $lc.offset(pos) + } + + var templateText = document.getElementById(template).innerHTML; + var rendered = Mustache.render(templateText, { + page: value.trim().replace(/\s+/g, '_'), + results: results + }, {}, ['[[', ']]']); + let selected = $lc.find('li.selected'); + if (selected) { + let selectedPos = $lc.find('li').index(selected[0]) + rendered = $(rendered) + const $lis = $lc.find('li') + if ($lis.length >= 1) { + selectedPos = Math.min(selectedPos, $lis.length - 1) + rendered.find('li')[selectedPos].classList.add('selected') + } + } + $lc.html(rendered).fadeIn() + return results + }) } $(document).on('keydown', '.keyboard-list', function (event) { @@ -422,18 +424,15 @@ if (holder) { if (insideSearch) { let query = value.substring(start + 1, end); - let searchTool = query => { - let result = commandSearch.search(query); + showSearchResults(commandSearch, query, input, value, 'command').then(results => { if (query.length > 0 && result.length === 0) { searchEnabled = false } - return result - } - showSearchResults(searchTool, query, input, value, 'command'); + }) return true } else if (insideLink) { let query = value.substring(start + 2, end); - showSearchResults(query => titleSearch.search(query), query, input, value, 'link'); + showSearchResults(titleSearch, query, input, value, 'link'); return true } else { $('#link-complete').fadeOut(); diff --git a/editor/src/search.js b/editor/src/search.js index 5655404..428dffe 100644 --- a/editor/src/search.js +++ b/editor/src/search.js @@ -1,45 +1,25 @@ -import lunr from 'lunr'; -import $ from 'jquery'; +import $ from 'jquery' +import qs from 'querystring'; function search(element) { return new Promise(function (resolve, reject) { - fetch('/documents.json') - .then(result => result.json()) - .then(documents => { - let mappedDocuments = {} - $.each(documents, function (index, doc) { - mappedDocuments[doc.url] = doc - }) - - let idx = lunr(function () { - this.ref('url') - this.field('title') - this.field('body') - - let lunridx = this - $.each(documents, function (index, doc) { - lunridx.add(doc) - }) - }) - - resolve({ - element: element, - idx: idx, - search(query) { - let result = this.idx.search(query) + resolve({ + element: element, + search(query) { + return fetch('/search/?' + qs.encode({q:query})) + .then(res => res.json()) + .then(data => { let actualResult = []; - - $.each(result, (key, value) => { + $.each(data.hits, (key, value) => { actualResult.push({ - ref: value.ref, - title: mappedDocuments[value.ref].title, + ref: value.id, + title: value.id.replace(/_/g, ' '), }) }) - return actualResult - } - }) - }) + }) + } + }) }) } diff --git a/file.go b/file.go index e0e6c11..3d05000 100644 --- a/file.go +++ b/file.go @@ -8,13 +8,13 @@ import ( "html" "html/template" "io/ioutil" - "log" "os" "os/exec" "path/filepath" "strings" "time" + "github.com/blevesearch/bleve" "github.com/sergi/go-diff/diffmatchpatch" ) @@ -33,10 +33,11 @@ type saveMessage struct { type FilePages struct { dirname string saveC chan saveMessage + index bleve.Index } -func NewFilePages(dirname string) PagesRepository { - fp := &FilePages{dirname, make(chan saveMessage)} +func NewFilePages(dirname string, index bleve.Index) PagesRepository { + fp := &FilePages{dirname, make(chan saveMessage), index} go func() { for msg := range fp.saveC { fp.save(msg) @@ -48,7 +49,7 @@ func NewFilePages(dirname string) PagesRepository { func (fp *FilePages) Get(title string) Page { name := strings.Replace(title, " ", "_", -1) title = strings.Replace(title, "_", " ", -1) - refs, err := loadBackrefs(fp, name) + refs, err := getBackrefs(fp, name) if err != nil { refs = nil } @@ -86,18 +87,16 @@ func (fp *FilePages) Save(p string, page Page, summary, author string) error { } func (fp *FilePages) save(msg saveMessage) error { - startTime := time.Now() - defer func() { - endTime := time.Now() - d := endTime.Sub(startTime) - log.Printf("Page saved in %s\n", d.String()) - }() - + var sw stopwatch + sw.Start("prepare") p := msg.p page := msg.page summary := msg.summary author := msg.author + page.Name = strings.Replace(p, " ", "_", -1) + page.Title = strings.Replace(p, "_", " ", -1) + f, err := os.Create(filepath.Join(fp.dirname, strings.Replace(p, " ", "_", -1))) if err != nil { return err @@ -109,7 +108,6 @@ func (fp *FilePages) save(msg saveMessage) error { if err != nil { return err } - _, err = buf.WriteTo(f) if err != nil { return err @@ -117,19 +115,68 @@ func (fp *FilePages) save(msg saveMessage) error { } else { f.WriteString(strings.Replace(page.Content, "\r\n", "\n", -1)) } - - err = processBackrefs(fp) + sw.Stop() + sw.Start("backrefs") + err = processBackrefs(fp.dirname, page) if err != nil { return fmt.Errorf("while processing backrefs: %s", err) } + sw.Stop() + sw.Start("git") err = saveWithGit(fp, p, summary, author) + if err != nil { + return fmt.Errorf("while saving to git: %w", err) + } + sw.Stop() + sw.Start("index") + err = fp.index.Index(page.Name, page) + if err != nil { + return fmt.Errorf("while indexing %s: %w", page.Name, err) + } + sw.Stop() + sw.Start("links") + err = saveLinksIncremental(fp.dirname, page.Title) + sw.Stop() + return err +} + +func saveLinksIncremental(dirname, title string) error { + type Document struct { + Title string `json:"title"` + } + var results []Document + f, err := os.Open(filepath.Join(dirname, LinksFile)) if err != nil { return err } - err = saveDocuments(fp) - err = saveLinks(fp) - return err + err = json.NewDecoder(f).Decode(&results) + if err != nil { + return err + } + f.Close() + + titles := make(map[string]bool) + for _, r := range results { + titles[r.Title] = true + } + + // Add new? title + titles[title] = true + + results = nil + for t, _ := range titles { + results = append(results, Document{t}) + } + + f, err = os.Create(filepath.Join(dirname, LinksFile)) + err = json.NewEncoder(f).Encode(&results) + if err != nil { + return err + } + f.Close() + + return nil } func saveLinks(fp *FilePages) error { @@ -144,7 +191,6 @@ func saveLinks(fp *FilePages) error { for _, page := range pages { results = append(results, Document{page.Title}) } - f, err := os.Create(filepath.Join(fp.dirname, LinksFile)) if err != nil { return err diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..8cab1f1 --- /dev/null +++ b/go.mod @@ -0,0 +1,24 @@ +module p83.nl/go/wiki + +go 1.14 + +require ( + github.com/RoaringBitmap/roaring v0.4.23 // indirect + github.com/blevesearch/bleve v1.0.9 + github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a // indirect + github.com/golang/protobuf v1.4.2 // indirect + github.com/sergi/go-diff v1.1.0 + github.com/stretchr/testify v1.4.0 + github.com/tinylib/msgp v1.1.2 // indirect + github.com/yuin/goldmark v1.1.32 + gitlab.com/golang-commonmark/linkify v0.0.0-20200225224916-64bca66f6ad3 // indirect + gitlab.com/golang-commonmark/markdown v0.0.0-20191127184510-91b5b3c99c19 + go.etcd.io/bbolt v1.3.5 // indirect + golang.org/x/net v0.0.0-20200625001655-4c5254603344 // indirect + golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect + golang.org/x/text v0.3.3 // indirect + google.golang.org/protobuf v1.25.0 // indirect + p83.nl/go/ekster v0.0.0-20191119211024-4511657daa0b + p83.nl/go/indieauth v0.1.0 + willnorris.com/go/microformats v1.1.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..3655efc --- /dev/null +++ b/go.sum @@ -0,0 +1,224 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= +github.com/RoaringBitmap/roaring v0.4.21 h1:WJ/zIlNX4wQZ9x8Ey33O1UaD9TCTakYsdLFSBcTwH+8= +github.com/RoaringBitmap/roaring v0.4.21/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= +github.com/RoaringBitmap/roaring v0.4.23 h1:gpyfd12QohbqhFO4NVDUdoPOCXsyahYRQhINmlHxKeo= +github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= +github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/blevesearch/bleve v1.0.9 h1:kqw/Ank/61UV9/Bx9kCcnfH6qWPgmS8O5LNfpsgzASg= +github.com/blevesearch/bleve v1.0.9/go.mod h1:tb04/rbU29clbtNgorgFd8XdJea4x3ybYaOjWKr+UBU= +github.com/blevesearch/blevex v0.0.0-20190916190636-152f0fe5c040/go.mod h1:WH+MU2F4T0VmSdaPX+Wu5GYoZBrYWdOZWSjzvYcDmqQ= +github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= +github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= +github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= +github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= +github.com/blevesearch/segment v0.9.0 h1:5lG7yBCx98or7gK2cHMKPukPZ/31Kag7nONpoBt22Ac= +github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ= +github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= +github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= +github.com/blevesearch/zap/v11 v11.0.9 h1:wlSrDBeGN1G4M51NQHIXca23ttwUfQpWaK7uhO5lRSo= +github.com/blevesearch/zap/v11 v11.0.9/go.mod h1:47hzinvmY2EvvJruzsSCJpro7so8L1neseaGjrtXHOY= +github.com/blevesearch/zap/v12 v12.0.9 h1:PpatkY+BLVFZf0Ok3/fwgI/I4RU0z5blXFGuQANmqXk= +github.com/blevesearch/zap/v12 v12.0.9/go.mod h1:paQuvxy7yXor+0Mx8p2KNmJgygQbQNN+W6HRfL5Hvwc= +github.com/blevesearch/zap/v13 v13.0.1 h1:NSCM6uKu77Vn/x9nlPp4pE1o/bftqcOWZEHSyZVpGBQ= +github.com/blevesearch/zap/v13 v13.0.1/go.mod h1:XmyNLMvMf8Z5FjLANXwUeDW3e1+o77TTGUWrth7T9WI= +github.com/blevesearch/zap/v14 v14.0.0 h1:HF8Ysjm13qxB0jTGaKLlatNXmJbQD8bY+PrPxm5v4hE= +github.com/blevesearch/zap/v14 v14.0.0/go.mod h1:sUc/gPGJlFbSQ2ZUh/wGRYwkKx+Dg/5p+dd+eq6QMXk= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k= +github.com/couchbase/moss v0.1.0/go.mod h1:9MaHIaRuy9pvLPUJxB8sh8OrLfyDczECVL37grCIubs= +github.com/couchbase/vellum v1.0.1 h1:qrj9ohvZedvc51S5KzPfJ6P6z0Vqzv7Lx7k3mVc2WOk= +github.com/couchbase/vellum v1.0.1/go.mod h1:FcwrEivFpNi24R3jLOs3n+fs5RnuQnQqCLBJ1uAg1W4= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 h1:Ujru1hufTHVb++eG6OuNDKMxZnGIvF6o/u8q/8h2+I4= +github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a h1:FQqoVvjbiUioBBFUL5up+h+GdCa/AnJsL/1bIs/veSI= +github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kljensen/snowball v0.6.0/go.mod h1:27N7E8fVU5H68RlUmnWwZCfxgt4POBJfENGMvNRhldw= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday v2.0.0+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/steveyen/gtreap v0.1.0 h1:CjhzTa274PyJLJuMZwIzCO1PfC00oRa8d1Kc78bFXJM= +github.com/steveyen/gtreap v0.1.0/go.mod h1:kl/5J7XbrOmlIbYIXdRHDDE5QxHqpk0cmkT7Z4dM9/Y= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU= +github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ= +github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc= +github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.32 h1:5tjfNdR2ki3yYQ842+eX2sQHeiwpKJ0RnHO4IYOc4V8= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +gitlab.com/golang-commonmark/html v0.0.0-20191124015941-a22733972181 h1:K+bMSIx9A7mLES1rtG+qKduLIXq40DAzYHtb0XuCukA= +gitlab.com/golang-commonmark/html v0.0.0-20191124015941-a22733972181/go.mod h1:dzYhVIwWCtzPAa4QP98wfB9+mzt33MSmM8wsKiMi2ow= +gitlab.com/golang-commonmark/linkify v0.0.0-20191026162114-a0c2df6c8f82 h1:oYrL81N608MLZhma3ruL8qTM4xcpYECGut8KSxRY59g= +gitlab.com/golang-commonmark/linkify v0.0.0-20191026162114-a0c2df6c8f82/go.mod h1:Gn+LZmCrhPECMD3SOKlE+BOHwhOYD9j7WT9NUtkCrC8= +gitlab.com/golang-commonmark/linkify v0.0.0-20200225224916-64bca66f6ad3 h1:1Coh5BsUBlXoEJmIEaNzVAWrtg9k7/eJzailMQr1grw= +gitlab.com/golang-commonmark/linkify v0.0.0-20200225224916-64bca66f6ad3/go.mod h1:Gn+LZmCrhPECMD3SOKlE+BOHwhOYD9j7WT9NUtkCrC8= +gitlab.com/golang-commonmark/markdown v0.0.0-20191127184510-91b5b3c99c19 h1:HsZm6XaTpEgZiZqcXZkUbG6BNtSZE3XyCTfo52YBoDY= +gitlab.com/golang-commonmark/markdown v0.0.0-20191127184510-91b5b3c99c19/go.mod h1:CRIzp0wh6PvKEAeEOtp9wEpNKJJ1VFTNfHO4+ToRgVA= +gitlab.com/golang-commonmark/mdurl v0.0.0-20191124015652-932350d1cb84 h1:qqjvoVXdWIcZCLPMlzgA7P9FZWdPGPvP/l3ef8GzV6o= +gitlab.com/golang-commonmark/mdurl v0.0.0-20191124015652-932350d1cb84/go.mod h1:IJZ+fdMvbW2qW6htJx7sLJ04FEs4Ldl/MDsJtMKywfw= +gitlab.com/golang-commonmark/puny v0.0.0-20191124015043-9f83538fa04f h1:Wku8eEdeJqIOFHtrfkYUByc4bCaTeA6fL0UJgfEiFMI= +gitlab.com/golang-commonmark/puny v0.0.0-20191124015043-9f83538fa04f/go.mod h1:Tiuhl+njh/JIg0uS/sOJVYi0x2HEa5rc1OAaVsb5tAs= +gitlab.com/opennota/wd v0.0.0-20180912061657-c5d65f63c638/go.mod h1:EGRJaqe2eO9XGmFtQCvV3Lm9NLico3UhFwUpCG/+mVU= +go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= +go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa h1:F+8P+gmewFQYRk6JoLQLwjBCTu3mcIURZfNkVweuRKA= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +p83.nl/go/ekster v0.0.0-20191119211024-4511657daa0b h1:MzIItcq83xZn1apSjCNovs55CZE/R/7GUnlOeIJ9HRI= +p83.nl/go/ekster v0.0.0-20191119211024-4511657daa0b/go.mod h1:NDCrcUlnixnkQ4u36jm3Hhvq0WcayEjg8SpAsFdQWLE= +p83.nl/go/indieauth v0.0.0-20180929170551-045fbfead8b8 h1:5YulOtUn7fpBZ+Rd0cPLUXo+Uhv375wozq9iduKUvcM= +p83.nl/go/indieauth v0.0.0-20180929170551-045fbfead8b8/go.mod h1:l5v2BkMmoICg0l4Wd+u104v5cpxI8cuYErA8ADkqCGI= +p83.nl/go/indieauth v0.1.0 h1:od0q6g15hH6bLKKdUjnj4wrXWhInUbsjXe8VgsDdh6Q= +p83.nl/go/indieauth v0.1.0/go.mod h1:YUg5PL1CpPlJHZv8zhFTNIp9pDDuEnGMXXYJpa9/1Ak= +willnorris.com/go/microformats v1.1.0 h1:a16gADl3aFxYVUQDxX8zS2AWAHKNnuaLlZFxyDzmSf8= +willnorris.com/go/microformats v1.1.0/go.mod h1:kvVnWrkkEscVAIITCEoiTX66Hcyg59C7q0E49mb9TJ0= diff --git a/main.go b/main.go index 979909a..91191ba 100644 --- a/main.go +++ b/main.go @@ -17,6 +17,7 @@ import ( "strings" "time" + "github.com/blevesearch/bleve" "p83.nl/go/ekster/pkg/util" "p83.nl/go/indieauth" ) @@ -81,6 +82,7 @@ type PagesRepository interface { Exist(p string) bool PageHistory(p string) ([]Revision, error) RecentChanges() ([]Change, error) + AllPages() ([]Page, error) } type pageBaseInfo struct { @@ -145,7 +147,9 @@ type recentPage struct { type indexHandler struct{} type graphHandler struct{} -type saveHandler struct{} +type saveHandler struct{ + SearchIndex bleve.Index +} type editHandler struct{} type historyHandler struct{} type recentHandler struct{} @@ -817,17 +821,46 @@ func main() { *baseurl = strings.TrimRight(*baseurl, "/") + "/" redirectURI = fmt.Sprintf("%sauth/callback", *baseurl) - mp = NewFilePages("data") + indexFilename := "index.bleve" + + var searchIndex bleve.Index + if _, err := os.Stat(indexFilename); os.IsNotExist(err) { + indexMapping := bleve.NewIndexMapping() + searchIndex, err = bleve.New(indexFilename, indexMapping) + if err != nil { + log.Fatal(err) + } + + pages, err := mp.AllPages() + if err != nil { + log.Fatal(err) + } + for _, page:= range pages { + err = searchIndex.Index(page.Name, page) + if err != nil { + log.Println(err) + } + } + } else { + searchIndex, err = bleve.Open(indexFilename) + if err != nil { + log.Fatal(err) + } + } + defer searchIndex.Close() + + sh, err := NewSearchHandler(searchIndex) + if err != nil { + log.Fatal(err) + } + + mp = NewFilePages("data", searchIndex) http.Handle("/auth/", &authHandler{}) http.HandleFunc("/links.json", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") http.ServeFile(w, r, filepath.Join(mp.(*FilePages).dirname, LinksFile)) }) - http.HandleFunc("/documents.json", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - http.ServeFile(w, r, filepath.Join(mp.(*FilePages).dirname, DocumentsFile)) - }) http.HandleFunc("/fetchLink", func(w http.ResponseWriter, r *http.Request) { link := r.URL.Query().Get("url") u, err := url.Parse(link) @@ -841,6 +874,7 @@ func main() { response.Meta.Title = "Test" json.NewEncoder(w).Encode(response) }) + http.Handle("/search/", sh) http.Handle("/public/", http.StripPrefix("/public/", http.FileServer(http.Dir("./dist")))) http.Handle("/save/", &saveHandler{}) http.Handle("/edit/", &editHandler{}) diff --git a/search.go b/search.go new file mode 100644 index 0000000..d1d1c0b --- /dev/null +++ b/search.go @@ -0,0 +1,41 @@ +package main + +import ( + "encoding/json" + "net/http" + + "github.com/blevesearch/bleve" + "github.com/blevesearch/bleve/mapping" +) + +// TODO: http handler +// TODO: index all pages on start +// TODO: reindex all command +// TODO: search(query) command + +type searchHandler struct { + documents PagesRepository + indexMapping mapping.IndexMapping + searchIndex bleve.Index +} + +func NewSearchHandler(searchIndex bleve.Index) (http.Handler, error) { + return &searchHandler{ + searchIndex: searchIndex, + }, nil +} + +func (s *searchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + q := bleve.NewQueryStringQuery(r.URL.Query().Get("q")) + sr := bleve.NewSearchRequest(q) + results, err := s.searchIndex.Search(sr) + if err != nil { + http.Error(w, err.Error(), 500) + } + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + err = enc.Encode(&results) + if err != nil { + http.Error(w, err.Error(), 500) + } +} diff --git a/util.go b/util.go index 6a12d95..8005c09 100644 --- a/util.go +++ b/util.go @@ -2,6 +2,7 @@ package main import ( "bufio" + "log" "math/rand" "regexp" "strings" @@ -69,3 +70,19 @@ func cleanNameURL(name string) string { func cleanTitle(name string) string { return strings.Replace(name, "_", " ", -1) } + +type stopwatch struct { + start time.Time + label string +} + +func (sw *stopwatch) Start(label string) { + sw.start = time.Now() + sw.label = label +} + +func (sw *stopwatch) Stop() { + endTime := time.Now() + d := endTime.Sub(sw.start) + log.Printf("%-20s: %s\n", sw.label, d.String()) +}