package main import ( "bufio" "bytes" "encoding/json" "errors" "fmt" "html" "html/template" "io/ioutil" "log" "os" "os/exec" "path/filepath" "strings" "time" "github.com/blevesearch/bleve" "github.com/sergi/go-diff/diffmatchpatch" ) const ( DocumentsFile = "_documents.json" LinksFile = "_links.json" BlocksDirectory = "_blocks" ) var BlockNotFound = errors.New("block not found") type saveMessage struct { p string page Page summary string author string } type Block struct { Text string Children []string Parent string } // ListItemV2 is way to convert from old structure to new structure type ListItemV2 struct { ID ID Indented int Text string } func (v2 ListItemV2) ListItem() ListItem { return ListItem{ v2.ID.StrID, v2.Indented, v2.Text, false, } } // ListItem is a simplification of the information that was saved by the editor type ListItem struct { ID string Indented int Text string Fleeting bool `json:"fleeting,omitempty"` } type ActualListItem struct { ID string `json:"id"` Indented int `json:"indented"` Text string `json:"text"` Fold string `json:"fold"` Hidden bool `json:"hidden"` Fleeting bool `json:"fleeting,omitempty"` } type FilePages struct { dirname string saveC chan saveMessage index bleve.Index } type BlockResponse struct { PageID string ParentID string Texts map[string]string Children map[string][]string Parents []string } func NewFilePages(dirname string, index bleve.Index) PagesRepository { err := os.MkdirAll(filepath.Join(dirname, "_blocks"), 0777) if err != nil { log.Fatalln(err) } fp := &FilePages{dirname, make(chan saveMessage), index} go func() { for msg := range fp.saveC { err := fp.save(msg) if err != nil { log.Println(err) } } }() return fp } func convertBlocksToListItems(current string, blocks BlockResponse, indent int) []ActualListItem { listItems := []ActualListItem{} for _, child := range blocks.Children[current] { l := convertBlocksToListItems(child, blocks, indent+1) listItems = append(listItems, ActualListItem{ ID: child, Indented: indent, Text: blocks.Texts[child], Fold: "open", // TODO: keep Fold state somewhere Hidden: false, }) listItems = append(listItems, l...) } return listItems } type titleOption struct { date bool timeObj time.Time } // 1_januari_2021 // 2021-01-01 func (fp *FilePages) Get(name string) Page { var sw stopwatch sw.Start("Get " + name) defer sw.Stop() var names []string var to titleOption pageNameDate, err := ParseDatePageName(name) if err == nil { to.date = true to.timeObj = pageNameDate names = append(names, name, pageNameDate.Format("2006-01-02")) } else if t, err := time.Parse("2006-01-02", name); err == nil { to.date = true to.timeObj = t names = append(names, formatDatePageName(to.timeObj), name) } else { names = append(names, name) } for _, name := range names { blocks, err := loadBlocks(fp.dirname, name) if err != nil && errors.Is(err, BlockNotFound) { continue } return fp.blocksBackendGet(name, blocks, to) } page, err := fp.oldPagesBackend(name) if err != nil { return fp.blocksBackendGet(name, BlockResponse{ParentID: "root", PageID: name}, to) } return page } func (fp *FilePages) blocksBackendGet(name string, blocks BlockResponse, option titleOption) Page { buf := bytes.Buffer{} current := blocks.PageID listItems := convertBlocksToListItems(current, blocks, 0) if listItems == nil { listItems = []ActualListItem{} } err := json.NewEncoder(&buf).Encode(&listItems) if err != nil { log.Printf("while encoding blocks: %s", err) } refs, err := getBackrefs(fp, name) if err != nil { refs = nil } title := formatTitle(blocks.Texts[name], option) if title == "" { title = cleanTitle(name) } return Page{ Name: name, Title: title, Content: buf.String(), Refs: refs, Blocks: blocks, Parent: blocks.ParentID, } } func formatTitle(title string, option titleOption) string { if option.date { return formatDateTitle(option.timeObj) } return title } func (fp *FilePages) oldPagesBackend(title string) (Page, error) { name := strings.Replace(title, " ", "_", -1) title = strings.Replace(title, "_", " ", -1) refs, err := getBackrefs(fp, name) if err != nil { refs = nil } f, err := os.Open(filepath.Join(fp.dirname, name)) if err != nil { log.Printf("while opening file in oldPagesBackend: %s", err) return Page{ Title: title, Name: name, Content: "", Refs: refs, }, err } defer f.Close() body, err := ioutil.ReadAll(f) if err != nil { log.Printf("while reading file in oldPagesBackend: %s", err) return Page{ Name: name, Title: title, Content: "", Refs: refs, }, err } return Page{ Name: name, Title: title, Content: string(body), Refs: refs, }, nil } func (fp *FilePages) Save(p string, page Page, summary, author string) error { fp.saveC <- saveMessage{p, page, summary, author} return nil } func (fp *FilePages) save(msg saveMessage) error { var sw stopwatch p := msg.p page := msg.page summary := msg.summary author := msg.author page.Name = strings.Replace(p, " ", "_", -1) page.Title = strings.Replace(p, "_", " ", -1) sw.Start("create blocks") err := saveBlocksFromPage(fp.dirname, page) if err != nil { log.Println(err) } sw.Stop() if p[0] != '_' { sw.Start("prepare") f, err := os.Create(filepath.Join(fp.dirname, strings.Replace(p, " ", "_", -1))) if err != nil { return err } defer f.Close() if page.Content[0] == '{' || page.Content[0] == '[' { var buf bytes.Buffer err = json.Indent(&buf, []byte(page.Content), "", " ") if err != nil { return err } _, err = buf.WriteTo(f) if err != nil { return err } } else { f.WriteString(strings.Replace(page.Content, "\r\n", "\n", -1)) } sw.Stop() sw.Start("backrefs") err = processBackrefs(fp.dirname, page) if err != nil { return fmt.Errorf("while processing backrefs: %s", err) } sw.Stop() sw.Start("git") err = saveWithGit(fp, p, summary, author) if err != nil { log.Printf("Error while saving to git: %w", err) // return fmt.Errorf("while saving to git: %w", err) } sw.Stop() sw.Start("index") searchObjects, err := createSearchObjects(page.Name) if err != nil { return fmt.Errorf("while creating search object %s: %w", page.Name, err) } for _, so := range searchObjects { if fp.index != nil { err = fp.index.Index(so.ID, so) if err != nil { return fmt.Errorf("while indexing %s: %w", page.Name, err) } } } sw.Stop() sw.Start("links") err = saveLinksIncremental(fp.dirname, page.Title) sw.Stop() } return nil } func saveWithNewIDs(dirname string, listItems []ListItemV2, pageName string) ([]ListItem, error) { var newListItems []ListItem for _, item := range listItems { newItem := ListItem{ ID: item.ID.NewID(), Indented: item.Indented, Text: item.Text, } newListItems = append(newListItems, newItem) } return newListItems, nil } func saveBlocksFromPage(dirname string, page Page) error { log.Println("Processing: ", page.Name) var listItems []ListItem err := json.NewDecoder(strings.NewReader(page.Content)).Decode(&listItems) if err != nil { log.Println("decoding default failed: %w", err) var listItemsV2 []ListItemV2 err = json.NewDecoder(strings.NewReader(page.Content)).Decode(&listItemsV2) listItems, err = saveWithNewIDs(dirname, listItemsV2, page.Name) if err != nil { return fmt.Errorf("while rewriting %s to use new ids: %w", page.Name, err) } } blocks := make(map[string]Block) prevList := make(map[string]ListItem) root := "root" parentBlock, err := loadBlock(dirname, page.Name) if err != nil { log.Println(err) } else { root = parentBlock.Parent } title := page.Title if page.Name[0] == '_' { title = parentBlock.Text } var parent = ListItem{ Text: title, Indented: -1, ID: page.Name, } prevList[parent.ID] = parent blocks[parent.ID] = Block{ Text: title, Children: []string{}, Parent: root, } var prev = &parent for i, item := range listItems { if item.Fleeting { continue } prevList[item.ID] = item if item.Indented > prev.Indented { parent = *prev } else if item.Indented == prev.Indented { // nothing } else if item.Indented <= parent.Indented { for item.Indented <= parent.Indented { if block, e := blocks[parent.ID]; e { parent = prevList[block.Parent] } } } blocks[item.ID] = Block{item.Text, []string{}, parent.ID} if block, e := blocks[parent.ID]; e { block.Children = append(block.Children, item.ID) blocks[parent.ID] = block } else { log.Println("Block missing") } prev = &listItems[i] } log.Printf("Loading parent block: %s", parent.ID) f, err := os.Open(filepath.Join(dirname, BlocksDirectory, parent.ID)) if err == nil { var parentBlock Block err = json.NewDecoder(f).Decode(&parentBlock) if err == nil { if pb, e := blocks[parent.ID]; e { pb.Text = parentBlock.Text pb.Parent = parentBlock.Parent blocks[parent.ID] = pb log.Printf("Text=%s, Parent=%s", parentBlock.Text, parentBlock.Parent) } } f.Close() } else { log.Println(err) err = nil } for id, block := range blocks { log.Println("Writing to ", id) f, err := os.OpenFile(filepath.Join(dirname, BlocksDirectory, id), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) if err != nil { log.Println(err) continue } enc := json.NewEncoder(f) enc.SetIndent("", " ") err = enc.Encode(&block) if err != nil { log.Println(err) } f.Close() } return err } func loadBlocks(dirname, rootBlockID string) (BlockResponse, error) { resp := BlockResponse{ rootBlockID, "", nil, nil, nil, } resp.Texts = make(map[string]string) resp.Children = make(map[string][]string) queue := []string{rootBlockID} block, err := loadBlock(dirname, rootBlockID) if err != nil { return BlockResponse{}, err } // NOTE: what does this do? if rootBlockID[0] != '_' && block.Children == nil { return BlockResponse{}, fmt.Errorf("not a block and has no children: %w", BlockNotFound) } prevID := rootBlockID parentID := block.Parent for parentID != "root" { parent, err := loadBlock(dirname, parentID) if err != nil { return BlockResponse{}, fmt.Errorf("while loading current parent block (%s->%s): %w", prevID, parentID, err) } resp.Texts[parentID] = parent.Text resp.Children[parentID] = parent.Children resp.ParentID = parentID resp.Parents = append(resp.Parents, parentID) prevID = parentID parentID = parent.Parent } if parentID == "root" { resp.ParentID = "root" } for { if len(queue) == 0 { break } current := queue[0] queue = queue[1:] block, err := loadBlock(dirname, current) if err != nil { return BlockResponse{}, err } resp.Texts[current] = block.Text resp.Children[current] = block.Children queue = append(queue, block.Children...) } return resp, nil } func loadBlock(dirname, blockID string) (Block, error) { f, err := os.Open(filepath.Join(dirname, BlocksDirectory, blockID)) if err != nil { return Block{}, fmt.Errorf("%q: %w", blockID, BlockNotFound) } defer f.Close() var block Block err = json.NewDecoder(f).Decode(&block) if err != nil { return Block{}, fmt.Errorf("%q: %v", blockID, err) } return block, nil } func saveLinksIncremental(dirname, title string) error { type Document struct { Title string `json:"title"` } var results []Document f, err := os.Open(filepath.Join(dirname, LinksFile)) if err != nil { return err } err = json.NewDecoder(f).Decode(&results) if err != nil { return err } f.Close() titles := make(map[string]bool) for _, r := range results { titles[r.Title] = true } // Add new? title titles[title] = true results = nil for t, _ := range titles { results = append(results, Document{t}) } f, err = os.Create(filepath.Join(dirname, LinksFile)) err = json.NewEncoder(f).Encode(&results) if err != nil { return err } f.Close() return nil } func saveLinks(mp PagesRepository) error { type Document struct { Title string `json:"title"` } var results []Document pages, err := mp.AllPages() if err != nil { return err } for _, page := range pages { results = append(results, Document{page.Title}) } f, err := os.Create(filepath.Join(mp.(*FilePages).dirname, LinksFile)) if err != nil { return err } defer f.Close() err = json.NewEncoder(f).Encode(&results) if err != nil { return err } return nil } func saveWithGit(fp *FilePages, p string, summary, author string) error { cmd := exec.Command("git", "add", ".") cmd.Dir = fp.dirname err := cmd.Run() if err != nil { return fmt.Errorf("while adding page %s: %s", p, err) } cmd = exec.Command("git", "commit", "-m", "Changes to "+p+" by "+author+"\n\n"+summary) cmd.Dir = fp.dirname // cmd.Stderr = os.Stderr // cmd.Stdout = os.Stderr err = cmd.Run() if err != nil { return fmt.Errorf("while commiting page %s: %s", p, err) } return nil } func (fp *FilePages) Exist(p string) bool { f, err := os.Open(filepath.Join(fp.dirname, strings.Replace(p, " ", "_", -1))) if err != nil { return os.IsExist(err) } f.Close() return true } func DiffPrettyHtml(diffs []diffmatchpatch.Diff) string { var buff bytes.Buffer for _, diff := range diffs { // text := strings.Replace(html.EscapeString(diff.Text), "\n", "
", -1) text := html.EscapeString(diff.Text) switch diff.Type { case diffmatchpatch.DiffInsert: _, _ = buff.WriteString("") _, _ = buff.WriteString(text) _, _ = buff.WriteString("") case diffmatchpatch.DiffDelete: _, _ = buff.WriteString("") _, _ = buff.WriteString(text) _, _ = buff.WriteString("") case diffmatchpatch.DiffEqual: _, _ = buff.WriteString("") _, _ = buff.WriteString(text) _, _ = buff.WriteString("") } } return buff.String() } func (fp *FilePages) PageHistory(p string) ([]Revision, error) { page := strings.Replace(p, " ", "_", -1) cmd := exec.Command("git", "log", "--pretty=oneline", "--no-decorate", "--color=never", page) cmd.Dir = fp.dirname output, err := cmd.StdoutPipe() if err != nil { return nil, err } defer output.Close() err = cmd.Start() if err != nil { return nil, fmt.Errorf("while starting: %s", err) } buf := bufio.NewScanner(output) var revisions []Revision for buf.Scan() { line := buf.Text() start := strings.Index(line, " ") commitId := line[0:start] rest := line[start+1:] pageText := gitRevision(fp.dirname, page, commitId) revisions = append(revisions, Revision{ Version: commitId, Page: DiffPage{Content: pageText}, Summary: rest, }) } dmp := diffmatchpatch.New() prevText := "" for i := len(revisions) - 1; i >= 0; i-- { diffs := dmp.DiffMain(prevText, revisions[i].Page.Content, false) revisions[i].Page.Diff = template.HTML(DiffPrettyHtml(diffs)) prevText = revisions[i].Page.Content } if err := cmd.Wait(); err != nil { return nil, fmt.Errorf("while waiting: %s", err) } return revisions, nil } func gitRevision(dirname, page, version string) string { cmd := exec.Command("git", "show", version+":"+page) cmd.Dir = dirname buf := bytes.Buffer{} cmd.Stdout = &buf cmd.Start() cmd.Wait() return buf.String() } func (fp *FilePages) RecentChanges() ([]Change, error) { cmd := exec.Command("git", "log", "--format=--1--%nDate: %aI%n--2--%n%b%n--3--", "--name-only") cmd.Dir = fp.dirname buf := bytes.Buffer{} cmd.Stdout = &buf err := cmd.Start() if err != nil { return nil, err } err = cmd.Wait() if err != nil { return nil, err } scanner := bufio.NewScanner(&buf) state := 0 var changes []Change var change Change body := "" for scanner.Scan() { line := scanner.Text() if line == "--1--" { state = 1 body = "" continue } if line == "--2--" { state = 2 continue } if line == "--3--" { state = 3 continue } if state == 1 && strings.HasPrefix(line, "Date: ") { line = line[6:] changeTime, err := time.Parse(time.RFC3339, line) if err != nil { return changes, err } change.Date = changeTime continue } if state == 2 { if line == "" { continue } body = body + line continue } if state == 3 { if line == "" { continue } change.Page = line } change.Body = body changes = append(changes, change) } return changes, nil } func (fp *FilePages) AllPages() ([]Page, error) { log.Println("AllPages", fp.dirname) files, err := ioutil.ReadDir(fp.dirname) if err != nil { return nil, err } var pages []Page for _, file := range files { if file.Name()[0] == '.' || file.Name()[0] == '_' { continue } if file.Name() == "backrefs.json" { continue } pages = append(pages, fp.Get(file.Name())) } return pages, nil }