435 lines
10 KiB
Go
435 lines
10 KiB
Go
/*
|
|
* Wiki - A wiki with editor
|
|
* Copyright (c) 2021 Peter Stuifzand
|
|
*
|
|
* This program is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"log"
|
|
"net/http"
|
|
"os"
|
|
"strings"
|
|
|
|
"github.com/blevesearch/bleve"
|
|
"github.com/blevesearch/bleve/mapping"
|
|
"github.com/iancoleman/strcase"
|
|
)
|
|
|
|
// TODO: http handler
|
|
// TODO: index all pages on start
|
|
// TODO: reindex all command
|
|
// TODO: search(query) command
|
|
|
|
type searchHandler struct {
|
|
indexMapping mapping.IndexMapping
|
|
searchIndex bleve.Index
|
|
}
|
|
|
|
type nameLine struct {
|
|
Name string `json:"name"`
|
|
Title string `json:"title"`
|
|
Line string `json:"line"`
|
|
}
|
|
|
|
type searchObject struct {
|
|
Title string `json:"title"`
|
|
Blocks []string `json:"blocks"`
|
|
Refs []nameLine `json:"refs"`
|
|
Meta map[string]interface{} `json:"meta"`
|
|
Links []ParsedLink `json:"links"`
|
|
}
|
|
|
|
func NewSearchHandler(searchIndex bleve.Index) (http.Handler, error) {
|
|
return &searchHandler{
|
|
searchIndex: searchIndex,
|
|
}, nil
|
|
}
|
|
|
|
func (s *searchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
defer r.Body.Close()
|
|
|
|
sess, err := NewSession(w, r)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 500)
|
|
return
|
|
}
|
|
defer func() {
|
|
if err := sess.Flush(); err != nil {
|
|
log.Println(err)
|
|
}
|
|
}()
|
|
if !sess.LoggedIn {
|
|
fmt.Fprint(w, "{}")
|
|
return
|
|
}
|
|
|
|
if r.URL.Query().Get("fields") == "1" {
|
|
fields, err := s.searchIndex.Fields()
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 500)
|
|
return
|
|
}
|
|
enc := json.NewEncoder(w)
|
|
enc.SetIndent("", " ")
|
|
err = enc.Encode(&fields)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 500)
|
|
return
|
|
}
|
|
return
|
|
} else if r.Method == "GET" && r.URL.Query().Get("reset") == "1" {
|
|
w.Header().Add("Content-Type", "text/html")
|
|
fmt.Fprint(w, `<form action="/search/" method="post"><input type=hidden name=reset value=1><input type=submit value="Reset Index"></form>`)
|
|
return
|
|
} else if r.Method == "POST" {
|
|
err = r.ParseForm()
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 500)
|
|
return
|
|
}
|
|
if r.PostForm.Get("reset") == "1" {
|
|
|
|
refs := make(Refs)
|
|
mp := NewFilePages("data", nil)
|
|
|
|
pages, err := mp.AllPages()
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 500)
|
|
return
|
|
}
|
|
|
|
for _, page := range pages {
|
|
err = saveBlocksFromPage("data", page)
|
|
if err != nil {
|
|
log.Printf("error while processing blocks from page %s: %v", page.Name, err)
|
|
continue
|
|
}
|
|
}
|
|
|
|
// Reload all pages
|
|
pages, err = mp.AllPages()
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 500)
|
|
return
|
|
}
|
|
|
|
for _, page := range pages {
|
|
log.Println("processing ", page.Title)
|
|
err = processBackrefsForPage(page, refs)
|
|
if err != nil {
|
|
log.Println("error while processing backrefs: ", err)
|
|
continue
|
|
}
|
|
}
|
|
|
|
log.Println("saveLinks")
|
|
err = saveLinks(mp)
|
|
if err != nil {
|
|
log.Printf("error while saving links %v", err)
|
|
http.Error(w, err.Error(), 500)
|
|
return
|
|
}
|
|
|
|
log.Println("saveBackrefs")
|
|
err = saveBackrefs("data/backrefs.json", refs)
|
|
if err != nil {
|
|
log.Printf("error while saving backrefs %v", err)
|
|
http.Error(w, err.Error(), 500)
|
|
return
|
|
}
|
|
|
|
err = os.RemoveAll("data/_tmp_index")
|
|
if err != nil {
|
|
log.Printf("error while remove old index %v", err)
|
|
http.Error(w, err.Error(), 500)
|
|
return
|
|
}
|
|
|
|
index, err := createSearchIndex("data", "_tmp_index")
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 500)
|
|
return
|
|
}
|
|
|
|
for _, page := range pages {
|
|
searchObjects, err := createSearchObjects(page.Name)
|
|
if err != nil {
|
|
log.Printf("error while creating search object %s: %v", page.Title, err)
|
|
continue
|
|
}
|
|
|
|
for _, so := range searchObjects {
|
|
err = index.Index(so.ID, so)
|
|
if err != nil {
|
|
log.Printf("error while indexing %s: %v", page.Title, err)
|
|
continue
|
|
}
|
|
}
|
|
}
|
|
|
|
err = os.Rename("data/_page-index", "data/_page-index-old")
|
|
if err != nil {
|
|
log.Printf("error while resetting index: %v", err)
|
|
http.Error(w, err.Error(), 500)
|
|
return
|
|
}
|
|
err = os.Rename("data/_tmp_index", "data/_page-index")
|
|
if err != nil {
|
|
log.Printf("error while putthing new index in place: %v", err)
|
|
http.Error(w, err.Error(), 500)
|
|
return
|
|
}
|
|
err = os.RemoveAll("data/_page-index-old")
|
|
if err != nil {
|
|
log.Printf("error while remove old index %v", err)
|
|
http.Error(w, err.Error(), 500)
|
|
return
|
|
}
|
|
|
|
enc := json.NewEncoder(w)
|
|
enc.SetIndent("", " ")
|
|
err = enc.Encode(struct {
|
|
Ok bool `json:"ok"`
|
|
}{Ok: true})
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 500)
|
|
return
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
q := bleve.NewQueryStringQuery(r.URL.Query().Get("q"))
|
|
sr := bleve.NewSearchRequest(q)
|
|
sr.IncludeLocations = false
|
|
sr.Size = 25
|
|
sr.Fields = []string{"page", "title", "text"}
|
|
sr.Highlight = bleve.NewHighlightWithStyle("html")
|
|
sr.Highlight.AddField("text")
|
|
results, err := s.searchIndex.Search(sr)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 500)
|
|
}
|
|
enc := json.NewEncoder(w)
|
|
enc.SetIndent("", " ")
|
|
err = enc.Encode(&results)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 500)
|
|
}
|
|
}
|
|
|
|
type pageBlock struct {
|
|
ID string `json:"id"`
|
|
Title string `json:"title"`
|
|
Page string `json:"page"`
|
|
Text string `json:"text"`
|
|
Link string `json:"link"`
|
|
}
|
|
|
|
func (p pageBlock) Type() string {
|
|
return "block"
|
|
}
|
|
|
|
func createSearchObjects(rootBlockID string) ([]pageBlock, error) {
|
|
blocks, err := loadBlocks("data", rootBlockID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var pageBlocks []pageBlock
|
|
|
|
queue := []string{blocks.PageID}
|
|
|
|
for len(queue) > 0 {
|
|
current := queue[0]
|
|
queue = queue[1:]
|
|
|
|
links, err := ParseLinks(current, blocks.Texts[current])
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
if len(links) == 0 {
|
|
pageBlocks = append(pageBlocks, pageBlock{
|
|
ID: current,
|
|
Title: blocks.Texts[blocks.PageID],
|
|
Page: blocks.PageID,
|
|
Text: blocks.Texts[current],
|
|
Link: "",
|
|
})
|
|
} else {
|
|
for _, link := range links {
|
|
pageBlocks = append(pageBlocks, pageBlock{
|
|
ID: current,
|
|
Title: blocks.Texts[blocks.PageID],
|
|
Page: blocks.PageID,
|
|
Text: blocks.Texts[current],
|
|
Link: link.Name,
|
|
})
|
|
}
|
|
}
|
|
|
|
queue = append(queue, blocks.Children[current]...)
|
|
}
|
|
|
|
return pageBlocks, nil
|
|
}
|
|
|
|
func createStructuredFormat(page Page) (searchObject, error) {
|
|
so := searchObject{}
|
|
so.Title = page.Title
|
|
so.Meta = make(map[string]interface{})
|
|
|
|
type simpleListItem struct {
|
|
Text string
|
|
ID string
|
|
Indented int
|
|
}
|
|
|
|
type parent struct {
|
|
key string
|
|
indent int
|
|
items []interface{}
|
|
values map[string]interface{}
|
|
}
|
|
|
|
var parents []parent
|
|
parents = append(parents, parent{
|
|
values: make(map[string]interface{}),
|
|
})
|
|
|
|
var listItems []simpleListItem
|
|
if err := json.NewDecoder(strings.NewReader(page.Content)).Decode(&listItems); err != nil {
|
|
so.Blocks = append(so.Blocks, page.Content)
|
|
} else {
|
|
for _, li := range listItems {
|
|
meta := strings.SplitN(li.Text, "::", 2)
|
|
par := parents[len(parents)-1]
|
|
// merge up
|
|
for len(parents) > 1 && li.Indented <= par.indent {
|
|
parents = parents[:len(parents)-1]
|
|
nextTop := parents[len(parents)-1]
|
|
if len(par.values) > 0 {
|
|
if vals, e := nextTop.values[par.key]; e {
|
|
if vals2, ok := vals.(map[string]interface{}); ok {
|
|
for k, v := range par.values {
|
|
vals2[k] = v
|
|
}
|
|
nextTop.values[par.key] = vals2
|
|
}
|
|
} else {
|
|
nextTop.values[par.key] = par.values
|
|
}
|
|
} else if len(par.items) > 0 {
|
|
nextTop.values[par.key] = par.items
|
|
} else {
|
|
nextTop.values[par.key] = ""
|
|
}
|
|
parents[len(parents)-1] = nextTop
|
|
par = parents[len(parents)-1]
|
|
}
|
|
if len(meta) == 2 {
|
|
key := strcase.ToSnake(strings.TrimSpace(meta[0]))
|
|
value := strings.TrimSpace(meta[1])
|
|
if value == "" {
|
|
parents = append(parents, parent{
|
|
key: key,
|
|
indent: li.Indented,
|
|
values: make(map[string]interface{}),
|
|
})
|
|
} else {
|
|
if len(parents) > 0 {
|
|
par = parents[len(parents)-1]
|
|
// save new value
|
|
if li.Indented > par.indent {
|
|
links, err := ParseLinks(li.ID, value)
|
|
if err != nil {
|
|
par.values[key] = value
|
|
} else {
|
|
if len(links) > 0 {
|
|
links[0].Href = fmt.Sprintf("%s%s", *baseurl, links[0].PageName)
|
|
links[0].ID = ""
|
|
par.values[key] = links[0]
|
|
} else {
|
|
par.values[key] = value
|
|
}
|
|
}
|
|
}
|
|
parents[len(parents)-1] = par
|
|
}
|
|
}
|
|
} else {
|
|
links, err := ParseLinks(li.ID, li.Text)
|
|
if err != nil {
|
|
par.items = append(par.items, li.Text)
|
|
} else if len(links) > 0 {
|
|
links[0].Href = fmt.Sprintf("%s%s", *baseurl, links[0].PageName)
|
|
links[0].ID = ""
|
|
par.items = append(par.items, links[0])
|
|
} else {
|
|
par.items = append(par.items, li.Text)
|
|
}
|
|
parents[len(parents)-1] = par
|
|
}
|
|
|
|
so.Blocks = append(so.Blocks, li.Text)
|
|
|
|
links, err := ParseLinks(li.ID, li.Text)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
for i, link := range links {
|
|
links[i].Href = fmt.Sprintf("%s%s", *baseurl, link.PageName)
|
|
}
|
|
|
|
so.Links = append(so.Links, links...)
|
|
}
|
|
}
|
|
|
|
// merge up
|
|
for len(parents) > 1 {
|
|
par := parents[len(parents)-1]
|
|
parents = parents[:len(parents)-1]
|
|
nextTop := parents[len(parents)-1]
|
|
if len(par.values) > 0 {
|
|
nextTop.values[par.key] = par.values
|
|
} else if len(par.items) > 0 {
|
|
nextTop.values[par.key] = par.items
|
|
} else {
|
|
nextTop.values[par.key] = ""
|
|
}
|
|
parents[len(parents)-1] = nextTop
|
|
}
|
|
|
|
so.Meta = parents[0].values
|
|
|
|
for _, refs := range page.Refs {
|
|
for _, ref := range refs {
|
|
so.Refs = append(so.Refs, nameLine{
|
|
ref.Name,
|
|
ref.Title,
|
|
strings.TrimSpace(ref.Line),
|
|
})
|
|
}
|
|
}
|
|
|
|
return so, nil
|
|
}
|