package main import ( "encoding/json" "fmt" "log" "net/http" "os" "strings" "github.com/blevesearch/bleve" "github.com/blevesearch/bleve/mapping" "github.com/iancoleman/strcase" ) // TODO: http handler // TODO: index all pages on start // TODO: reindex all command // TODO: search(query) command type searchHandler struct { indexMapping mapping.IndexMapping searchIndex bleve.Index } type nameLine struct { Name string `json:"name"` Title string `json:"title"` Line string `json:"line"` } type searchObject struct { Title string `json:"title"` Blocks []string `json:"blocks"` Refs []nameLine `json:"refs"` Meta map[string]string `json:"meta"` Links []ParsedLink `json:"links"` } func NewSearchHandler(searchIndex bleve.Index) (http.Handler, error) { return &searchHandler{ searchIndex: searchIndex, }, nil } func (s *searchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() sess, err := NewSession(w, r) if err != nil { http.Error(w, err.Error(), 500) return } defer func() { if err := sess.Flush(); err != nil { log.Println(err) } }() if !sess.LoggedIn { fmt.Fprint(w, "{}") return } if r.URL.Query().Get("fields") == "1" { fields, err := s.searchIndex.Fields() if err != nil { http.Error(w, err.Error(), 500) return } enc := json.NewEncoder(w) enc.SetIndent("", " ") err = enc.Encode(&fields) if err != nil { http.Error(w, err.Error(), 500) return } return } else if r.Method == "GET" && r.URL.Query().Get("reset") == "1" { w.Header().Add("Content-Type", "text/html") fmt.Fprint(w, `
`) return } else if r.Method == "POST" { err = r.ParseForm() if err != nil { http.Error(w, err.Error(), 500) return } if r.PostForm.Get("reset") == "1" { refs := make(Refs) mp := NewFilePages("data", nil) pages, err := mp.AllPages() if err != nil { http.Error(w, err.Error(), 500) return } for _, page := range pages { err = saveBlocksFromPage("data", page) if err != nil { log.Printf("error while processing blocks from page %s: %w", page.Name, err) continue } } // Reload all pages pages, err = mp.AllPages() if err != nil { http.Error(w, err.Error(), 500) return } for _, page := range pages { log.Println("processing ", page.Title) err = processBackrefsForPage(page, refs) if err != nil { log.Println("error while processing backrefs: ", err) continue } } log.Println("saveLinks") err = saveLinks(mp) if err != nil { log.Printf("error while saving links %w", err) http.Error(w, err.Error(), 500) return } log.Println("saveBackrefs") err = saveBackrefs("data/backrefs.json", refs) if err != nil { log.Printf("error while saving backrefs %w", err) http.Error(w, err.Error(), 500) return } err = os.RemoveAll("data/_tmp_index") if err != nil { log.Printf("error while remove old index %w", err) http.Error(w, err.Error(), 500) return } index, err := createSearchIndex("data", "_tmp_index") if err != nil { http.Error(w, err.Error(), 500) return } for _, page := range pages { searchObjects, err := createSearchObjects(page.Name) if err != nil { log.Printf("error while creating search object %s: %w", page.Title, err) continue } for _, so := range searchObjects { err = index.Index(so.ID, so) if err != nil { log.Printf("error while indexing %s: %w", page.Title, err) continue } } } err = os.Rename("data/_page-index", "data/_page-index-old") if err != nil { log.Printf("error while resetting index: %w", err) http.Error(w, err.Error(), 500) return } err = os.Rename("data/_tmp_index", "data/_page-index") if err != nil { log.Printf("error while putthing new index in place: %w", err) http.Error(w, err.Error(), 500) return } err = os.RemoveAll("data/_page-index-old") if err != nil { log.Printf("error while remove old index %w", err) http.Error(w, err.Error(), 500) return } enc := json.NewEncoder(w) enc.SetIndent("", " ") err = enc.Encode(struct { Ok bool `json:"ok"` }{Ok: true}) if err != nil { http.Error(w, err.Error(), 500) return } } return } q := bleve.NewQueryStringQuery(r.URL.Query().Get("q")) sr := bleve.NewSearchRequest(q) sr.IncludeLocations = false sr.Size = 25 sr.Fields = []string{"page", "title", "text"} sr.Highlight = bleve.NewHighlightWithStyle("html") sr.Highlight.AddField("text") results, err := s.searchIndex.Search(sr) if err != nil { http.Error(w, err.Error(), 500) } enc := json.NewEncoder(w) enc.SetIndent("", " ") err = enc.Encode(&results) if err != nil { http.Error(w, err.Error(), 500) } } type pageBlock struct { ID string `json:"id"` Title string `json:"title"` Page string `json:"page"` Text string `json:"text"` Link string `json:"link"` } func (p pageBlock) Type() string { return "block" } func createSearchObjects(rootBlockID string) ([]pageBlock, error) { blocks, err := loadBlocks("data", rootBlockID) if err != nil { return nil, err } var pageBlocks []pageBlock queue := []string{blocks.PageID} for len(queue) > 0 { current := queue[0] queue = queue[1:] links, err := ParseLinks(current, blocks.Texts[current]) if err != nil { continue } if len(links) == 0 { pageBlocks = append(pageBlocks, pageBlock{ ID: current, Title: blocks.Texts[blocks.PageID], Page: blocks.PageID, Text: blocks.Texts[current], Link: "", }) } else { for _, link := range links { pageBlocks = append(pageBlocks, pageBlock{ ID: current, Title: blocks.Texts[blocks.PageID], Page: blocks.PageID, Text: blocks.Texts[current], Link: link.Name, }) } } queue = append(queue, blocks.Children[current]...) } return pageBlocks, nil } func createStructuredFormat(page Page) (searchObject, error) { so := searchObject{} so.Title = page.Title so.Meta = make(map[string]string) type simpleListItem struct { Text string ID string } var listItems []simpleListItem if err := json.NewDecoder(strings.NewReader(page.Content)).Decode(&listItems); err != nil { so.Blocks = append(so.Blocks, page.Content) } else { for _, li := range listItems { meta := strings.SplitN(li.Text, "::", 2) if len(meta) == 2 { key := strcase.ToSnake(strings.TrimSpace(meta[0])) value := strings.TrimSpace(meta[1]) if key == "title" { so.Title = value } so.Meta[key] = value } so.Blocks = append(so.Blocks, li.Text) links, err := ParseLinks(li.ID, li.Text) if err != nil { continue } for i, link := range links { links[i].Href = fmt.Sprintf("%s%s", *baseurl, link.PageName) } so.Links = append(so.Links, links...) } } for _, refs := range page.Refs { for _, ref := range refs { so.Refs = append(so.Refs, nameLine{ ref.Name, ref.Title, strings.TrimSpace(ref.Line), }) } } return so, nil }