|
|
|
@ -25,9 +25,11 @@ import (
|
|
|
|
|
"net/http"
|
|
|
|
|
"os"
|
|
|
|
|
"strings"
|
|
|
|
|
"time"
|
|
|
|
|
|
|
|
|
|
"github.com/blevesearch/bleve/v2"
|
|
|
|
|
"github.com/blevesearch/bleve/v2/mapping"
|
|
|
|
|
"github.com/davecgh/go-spew/spew"
|
|
|
|
|
"github.com/iancoleman/strcase"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
@ -53,6 +55,7 @@ type searchObject struct {
|
|
|
|
|
Refs []nameLine `json:"refs"`
|
|
|
|
|
Meta map[string]interface{} `json:"meta"`
|
|
|
|
|
Links []ParsedLink `json:"links"`
|
|
|
|
|
Dates []time.Time `json:"dates"`
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func NewSearchHandler(searchIndex bleve.Index) (http.Handler, error) {
|
|
|
|
@ -104,6 +107,9 @@ func (s *searchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
if r.PostForm.Get("reset") == "1" {
|
|
|
|
|
var sw stopwatch
|
|
|
|
|
sw.Start("full reset")
|
|
|
|
|
defer sw.Stop()
|
|
|
|
|
|
|
|
|
|
refs := make(Refs)
|
|
|
|
|
mp := NewFilePages("data", nil)
|
|
|
|
@ -122,6 +128,8 @@ func (s *searchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
sw.Lap("save blocks from pages")
|
|
|
|
|
|
|
|
|
|
// Reload all pages
|
|
|
|
|
pages, err = mp.AllPages()
|
|
|
|
|
if err != nil {
|
|
|
|
@ -138,21 +146,23 @@ func (s *searchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
log.Println("saveLinks")
|
|
|
|
|
sw.Lap("process backrefs for pages")
|
|
|
|
|
|
|
|
|
|
err = saveLinks(mp)
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Printf("error while saving links %v", err)
|
|
|
|
|
http.Error(w, err.Error(), 500)
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
sw.Lap("save links")
|
|
|
|
|
|
|
|
|
|
log.Println("saveBackrefs")
|
|
|
|
|
err = saveBackrefs("data/backrefs.json", refs)
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Printf("error while saving backrefs %v", err)
|
|
|
|
|
http.Error(w, err.Error(), 500)
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
sw.Lap("save backrefs")
|
|
|
|
|
|
|
|
|
|
err = os.RemoveAll("data/_tmp_index")
|
|
|
|
|
if err != nil {
|
|
|
|
@ -161,28 +171,12 @@ func (s *searchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
index, err := createSearchIndex("data", "_tmp_index")
|
|
|
|
|
_, err = createSearchIndex("data", "_tmp_index")
|
|
|
|
|
if err != nil {
|
|
|
|
|
http.Error(w, err.Error(), 500)
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for _, page := range pages {
|
|
|
|
|
searchObjects, err := createSearchObjects(page.Name)
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Printf("error while creating search object %s: %v", page.Title, err)
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for _, so := range searchObjects {
|
|
|
|
|
err = index.Index(so.ID, so)
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Printf("error while indexing %s: %v", page.Title, err)
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
err = os.Rename("data/_page-index", "data/_page-index-old")
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Printf("error while resetting index: %v", err)
|
|
|
|
@ -201,6 +195,7 @@ func (s *searchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
|
|
|
http.Error(w, err.Error(), 500)
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
sw.Lap("indexing")
|
|
|
|
|
|
|
|
|
|
enc := json.NewEncoder(w)
|
|
|
|
|
enc.SetIndent("", " ")
|
|
|
|
@ -219,7 +214,7 @@ func (s *searchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
|
|
|
sr := bleve.NewSearchRequest(q)
|
|
|
|
|
sr.IncludeLocations = false
|
|
|
|
|
sr.Size = 25
|
|
|
|
|
sr.Fields = []string{"page", "title", "text"}
|
|
|
|
|
sr.Fields = []string{"page", "title", "text", "date"}
|
|
|
|
|
sr.Highlight = bleve.NewHighlightWithStyle("html")
|
|
|
|
|
sr.Highlight.AddField("text")
|
|
|
|
|
results, err := s.searchIndex.Search(sr)
|
|
|
|
@ -235,11 +230,13 @@ func (s *searchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type pageBlock struct {
|
|
|
|
|
ID string `json:"id"`
|
|
|
|
|
Title string `json:"title"`
|
|
|
|
|
Page string `json:"page"`
|
|
|
|
|
Text string `json:"text"`
|
|
|
|
|
Link string `json:"link"`
|
|
|
|
|
ID string `json:"id"`
|
|
|
|
|
Title string `json:"title"`
|
|
|
|
|
Page string `json:"page"`
|
|
|
|
|
Text string `json:"text"`
|
|
|
|
|
Link []string `json:"link"`
|
|
|
|
|
Tag []string `json:"tag"`
|
|
|
|
|
Date []time.Time `json:"date"`
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (p pageBlock) Type() string {
|
|
|
|
@ -247,6 +244,7 @@ func (p pageBlock) Type() string {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func createSearchObjects(rootBlockID string) ([]pageBlock, error) {
|
|
|
|
|
log.Println("createSearchObjects", rootBlockID)
|
|
|
|
|
blocks, err := loadBlocks("data", rootBlockID)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
@ -262,32 +260,49 @@ func createSearchObjects(rootBlockID string) ([]pageBlock, error) {
|
|
|
|
|
|
|
|
|
|
links, err := ParseLinks(current, blocks.Texts[current])
|
|
|
|
|
if err != nil {
|
|
|
|
|
continue
|
|
|
|
|
log.Println("ParseLinks", err)
|
|
|
|
|
links = nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if len(links) == 0 {
|
|
|
|
|
pageBlocks = append(pageBlocks, pageBlock{
|
|
|
|
|
ID: current,
|
|
|
|
|
Title: blocks.Texts[blocks.PageID],
|
|
|
|
|
Page: blocks.PageID,
|
|
|
|
|
Text: blocks.Texts[current],
|
|
|
|
|
Link: "",
|
|
|
|
|
})
|
|
|
|
|
} else {
|
|
|
|
|
for _, link := range links {
|
|
|
|
|
pageBlocks = append(pageBlocks, pageBlock{
|
|
|
|
|
ID: current,
|
|
|
|
|
Title: blocks.Texts[blocks.PageID],
|
|
|
|
|
Page: blocks.PageID,
|
|
|
|
|
Text: blocks.Texts[current],
|
|
|
|
|
Link: link.Name,
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
var linkNames []string
|
|
|
|
|
for _, link := range links {
|
|
|
|
|
linkNames = append(linkNames, link.Name)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
tags, err := ParseTags(blocks.Texts[current])
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Println("ParseTags", err)
|
|
|
|
|
tags = nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
dates, err := ParseDates(blocks.Texts[current])
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Println("ParseDates", err)
|
|
|
|
|
dates = nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pageDate, err := ParseDatePageName(blocks.Texts[blocks.PageID])
|
|
|
|
|
if err == nil {
|
|
|
|
|
dates = append(dates, pageDate)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pageBlocks = append(pageBlocks, pageBlock{
|
|
|
|
|
ID: current,
|
|
|
|
|
Title: blocks.Texts[blocks.PageID],
|
|
|
|
|
Page: blocks.PageID,
|
|
|
|
|
Text: blocks.Texts[current],
|
|
|
|
|
Link: linkNames,
|
|
|
|
|
Tag: tags,
|
|
|
|
|
Date: dates,
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
queue = append(queue, blocks.Children[current]...)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if rootBlockID == "Henk_Stuifzand" {
|
|
|
|
|
spew.Dump(pageBlocks)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return pageBlocks, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@ -400,9 +415,19 @@ func createStructuredFormat(page Page) (searchObject, error) {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
so.Links = append(so.Links, links...)
|
|
|
|
|
|
|
|
|
|
dates, err := ParseDates(li.Text)
|
|
|
|
|
if err != nil {
|
|
|
|
|
dates = nil
|
|
|
|
|
}
|
|
|
|
|
so.Dates = append(so.Dates, dates...)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
date, err := ParseDatePageName(so.Title)
|
|
|
|
|
if err == nil {
|
|
|
|
|
so.Dates = append(so.Dates, date)
|
|
|
|
|
}
|
|
|
|
|
// merge up
|
|
|
|
|
for len(parents) > 1 {
|
|
|
|
|
par := parents[len(parents)-1]
|
|
|
|
|