/* * Wiki - A wiki with editor * Copyright (c) 2021 Peter Stuifzand * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ package main import ( "encoding/json" "fmt" "log" "net/http" "os" "strings" "time" "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/mapping" "github.com/iancoleman/strcase" ) // TODO: http handler // TODO: index all pages on start // TODO: reindex all command // TODO: search(query) command type searchHandler struct { indexMapping mapping.IndexMapping searchIndex bleve.Index } type nameLine struct { Name string `json:"name"` Title string `json:"title"` Line string `json:"line"` } type searchObject struct { Title string `json:"title"` Blocks []string `json:"blocks"` Refs []nameLine `json:"refs"` Meta map[string]interface{} `json:"meta"` Links []ParsedLink `json:"links"` Dates []time.Time `json:"dates"` } func NewSearchHandler(searchIndex bleve.Index) (http.Handler, error) { return &searchHandler{ searchIndex: searchIndex, }, nil } func (s *searchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() sess, err := NewSession(w, r) if err != nil { http.Error(w, err.Error(), 500) return } defer func() { if err := sess.Flush(); err != nil { log.Println(err) } }() if !sess.LoggedIn { fmt.Fprint(w, "{}") return } if r.URL.Query().Get("fields") == "1" { fields, err := s.searchIndex.Fields() if err != nil { http.Error(w, err.Error(), 500) return } enc := json.NewEncoder(w) enc.SetIndent("", " ") err = enc.Encode(&fields) if err != nil { http.Error(w, err.Error(), 500) return } return } else if r.Method == "GET" && r.URL.Query().Get("reset") == "1" { w.Header().Add("Content-Type", "text/html") fmt.Fprint(w, `
`) return } else if r.Method == "POST" { err = r.ParseForm() if err != nil { http.Error(w, err.Error(), 500) return } if r.PostForm.Get("reset") == "1" { var sw stopwatch sw.Start("full reset") defer sw.Stop() refs := make(Refs) mp := NewFilePages("data", nil) pages, err := mp.AllPages() if err != nil { http.Error(w, err.Error(), 500) return } for _, page := range pages { err = saveBlocksFromPage("data", page) if err != nil { log.Printf("error while processing blocks from page %s: %v", page.Name, err) continue } } sw.Lap("save blocks from pages") // Reload all pages pages, err = mp.AllPages() if err != nil { http.Error(w, err.Error(), 500) return } for _, page := range pages { log.Println("processing ", page.Title) err = processBackrefsForPage(page, refs) if err != nil { log.Println("error while processing backrefs: ", err) continue } } sw.Lap("process backrefs for pages") err = saveLinks(mp) if err != nil { log.Printf("error while saving links %v", err) http.Error(w, err.Error(), 500) return } sw.Lap("save links") err = saveBackrefs("data/backrefs.json", refs) if err != nil { log.Printf("error while saving backrefs %v", err) http.Error(w, err.Error(), 500) return } sw.Lap("save backrefs") err = os.RemoveAll("data/_tmp_index") if err != nil { log.Printf("error while remove old index %v", err) http.Error(w, err.Error(), 500) return } _, err = createSearchIndex("data", "_tmp_index") if err != nil { http.Error(w, err.Error(), 500) return } err = os.Rename("data/_page-index", "data/_page-index-old") if err != nil { log.Printf("error while resetting index: %v", err) http.Error(w, err.Error(), 500) return } err = os.Rename("data/_tmp_index", "data/_page-index") if err != nil { log.Printf("error while putthing new index in place: %v", err) http.Error(w, err.Error(), 500) return } err = os.RemoveAll("data/_page-index-old") if err != nil { log.Printf("error while remove old index %v", err) http.Error(w, err.Error(), 500) return } sw.Lap("indexing") enc := json.NewEncoder(w) enc.SetIndent("", " ") err = enc.Encode(struct { Ok bool `json:"ok"` }{Ok: true}) if err != nil { http.Error(w, err.Error(), 500) return } } return } q := bleve.NewQueryStringQuery(r.URL.Query().Get("q")) sr := bleve.NewSearchRequest(q) sr.IncludeLocations = false sr.Size = 25 sr.Fields = []string{"page", "title", "text", "date", "parent"} sr.Highlight = bleve.NewHighlightWithStyle("html") sr.Highlight.AddField("text") results, err := s.searchIndex.Search(sr) if err != nil { http.Error(w, err.Error(), 500) } enc := json.NewEncoder(w) enc.SetIndent("", " ") err = enc.Encode(&results) if err != nil { http.Error(w, err.Error(), 500) } } type pageBlock struct { ID string `json:"id"` Parent string `json:"parent"` Title string `json:"title"` Page string `json:"page"` Text string `json:"text"` Link []string `json:"link"` Tag []string `json:"tag"` Date []time.Time `json:"date"` Key string `json:"key"` Value string `json:"value"` } func (p pageBlock) Type() string { return "block" } func createSearchObjects(rootBlockID string) ([]pageBlock, error) { log.Println("createSearchObjects", rootBlockID) blocks, err := loadBlocks("data", rootBlockID) if err != nil { return nil, err } if len(blocks.Parents) > 0 { page := blocks.Parents[len(blocks.Parents)-1] if page != rootBlockID { blocks, err = loadBlocks("data", page) } } var pageBlocks []pageBlock queue := []string{blocks.PageID} for len(queue) > 0 { current := queue[0] queue = queue[1:] links, err := ParseLinks(current, blocks.Texts[current]) if err != nil { log.Println("ParseLinks", err) links = nil } var linkNames []string for _, link := range links { linkNames = append(linkNames, link.Name) } tags, err := ParseTags(blocks.Texts[current]) if err != nil { log.Println("ParseTags", err) tags = nil } dates, err := ParseDates(blocks.Texts[current]) if err != nil { log.Println("ParseDates", err) dates = nil } pageDate, err := ParseDatePageName(blocks.Texts[blocks.PageID]) if err == nil { dates = append(dates, pageDate) } block := pageBlock{ ID: current, Parent: blocks.ParentID, Title: blocks.Texts[blocks.PageID], Page: blocks.PageID, Text: blocks.Texts[current], Link: linkNames, Tag: tags, Date: dates, } if kvpair := strings.SplitN(blocks.Texts[current], "::", 2); len(kvpair) == 2 { block.Key = strings.TrimSpace(kvpair[0]) block.Value = strings.TrimSpace(kvpair[1]) } pageBlocks = append(pageBlocks, block) queue = append(queue, blocks.Children[current]...) } return pageBlocks, nil } func createStructuredFormat(page Page) (searchObject, error) { so := searchObject{} so.Title = page.Title so.Meta = make(map[string]interface{}) type simpleListItem struct { Text string ID string Indented int } type parent struct { key string indent int items []interface{} values map[string]interface{} } var parents []parent parents = append(parents, parent{ values: make(map[string]interface{}), }) var listItems []simpleListItem if err := json.NewDecoder(strings.NewReader(page.Content)).Decode(&listItems); err != nil { so.Blocks = append(so.Blocks, page.Content) } else { for _, li := range listItems { meta := strings.SplitN(li.Text, "::", 2) par := parents[len(parents)-1] // merge up for len(parents) > 1 && li.Indented <= par.indent { parents = parents[:len(parents)-1] nextTop := parents[len(parents)-1] if len(par.values) > 0 { if vals, e := nextTop.values[par.key]; e { if vals2, ok := vals.(map[string]interface{}); ok { for k, v := range par.values { vals2[k] = v } nextTop.values[par.key] = vals2 } } else { nextTop.values[par.key] = par.values } } else if len(par.items) > 0 { nextTop.values[par.key] = par.items } else { nextTop.values[par.key] = "" } parents[len(parents)-1] = nextTop par = parents[len(parents)-1] } if len(meta) == 2 { key := strcase.ToSnake(strings.TrimSpace(meta[0])) value := strings.TrimSpace(meta[1]) if value == "" { parents = append(parents, parent{ key: key, indent: li.Indented, values: make(map[string]interface{}), }) } else { if len(parents) > 0 { par = parents[len(parents)-1] // save new value if li.Indented > par.indent { links, err := ParseLinks(li.ID, value) if err != nil { par.values[key] = value } else { if len(links) > 0 { links[0].Href = fmt.Sprintf("%s%s", *baseurl, links[0].PageName) links[0].ID = "" par.values[key] = links[0] } else { par.values[key] = value } } } parents[len(parents)-1] = par } } } else { links, err := ParseLinks(li.ID, li.Text) if err != nil { par.items = append(par.items, li.Text) } else if len(links) > 0 { links[0].Href = fmt.Sprintf("%s%s", *baseurl, links[0].PageName) links[0].ID = "" par.items = append(par.items, links[0]) } else { par.items = append(par.items, li.Text) } parents[len(parents)-1] = par } so.Blocks = append(so.Blocks, li.Text) links, err := ParseLinks(li.ID, li.Text) if err != nil { continue } for i, link := range links { links[i].Href = fmt.Sprintf("%s%s", *baseurl, link.PageName) } so.Links = append(so.Links, links...) dates, err := ParseDates(li.Text) if err != nil { dates = nil } so.Dates = append(so.Dates, dates...) } } date, err := ParseDatePageName(so.Title) if err == nil { so.Dates = append(so.Dates, date) } // merge up for len(parents) > 1 { par := parents[len(parents)-1] parents = parents[:len(parents)-1] nextTop := parents[len(parents)-1] if len(par.values) > 0 { nextTop.values[par.key] = par.values } else if len(par.items) > 0 { nextTop.values[par.key] = par.items } else { nextTop.values[par.key] = "" } parents[len(parents)-1] = nextTop } so.Meta = parents[0].values for _, refs := range page.Refs { for _, ref := range refs { so.Refs = append(so.Refs, nameLine{ ref.Name, ref.Title, strings.TrimSpace(ref.Line), }) } } return so, nil }