You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
wiki/file.go

820 lines
18 KiB

/*
* Wiki - A wiki with editor
* Copyright (c) 2021-2021 Peter Stuifzand
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package main
import (
"bufio"
"bytes"
"encoding/json"
"errors"
"fmt"
"html"
"html/template"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/blevesearch/bleve"
"github.com/sergi/go-diff/diffmatchpatch"
)
const (
DocumentsFile = "_documents.json"
LinksFile = "_links.json"
BlocksDirectory = "_blocks"
)
var BlockNotFound = errors.New("block not found")
type saveMessage struct {
p string
page Page
summary string
author string
}
type Block struct {
Text string
Children []string
Parent string
}
// ListItemV2 is way to convert from old structure to new structure
type ListItemV2 struct {
ID ID
Indented int
Text string
}
func (v2 ListItemV2) ListItem() ListItem {
return ListItem{
v2.ID.StrID,
v2.Indented,
v2.Text,
false,
}
}
// ListItem is a simplification of the information that was saved by the editor
type ListItem struct {
ID string
Indented int
Text string
Fleeting bool `json:"fleeting,omitempty"`
}
type ActualListItem struct {
ID string `json:"id"`
Indented int `json:"indented"`
Text string `json:"text"`
Fold string `json:"fold"`
Hidden bool `json:"hidden"`
Fleeting bool `json:"fleeting,omitempty"`
}
type FilePages struct {
dirname string
saveC chan saveMessage
index bleve.Index
}
type BlockResponse struct {
PageID string
ParentID string
Texts map[string]string
Children map[string][]string
Parents []string
}
func NewFilePages(dirname string, index bleve.Index) PagesRepository {
err := os.MkdirAll(filepath.Join(dirname, "_blocks"), 0777)
if err != nil {
log.Fatalln(err)
}
fp := &FilePages{dirname, make(chan saveMessage), index}
go func() {
for msg := range fp.saveC {
err := fp.save(msg)
if err != nil {
log.Println(err)
}
}
}()
return fp
}
func convertBlocksToListItems(current string, blocks BlockResponse, indent int) []ActualListItem {
listItems := []ActualListItem{}
for _, child := range blocks.Children[current] {
l := convertBlocksToListItems(child, blocks, indent+1)
listItems = append(listItems,
ActualListItem{
ID: child,
Indented: indent,
Text: blocks.Texts[child],
Fold: "open", // TODO: keep Fold state somewhere
Hidden: false,
})
listItems = append(listItems, l...)
}
return listItems
}
type titleOption struct {
date bool
timeObj time.Time
}
// 1_januari_2021
// 2021-01-01
func (fp *FilePages) Get(name string) Page {
var sw stopwatch
sw.Start("Get " + name)
defer sw.Stop()
var names []string
var to titleOption
pageNameDate, err := ParseDatePageName(name)
if err == nil {
to.date = true
to.timeObj = pageNameDate
names = append(names, name, pageNameDate.Format("2006-01-02"))
} else if t, err := time.Parse("2006-01-02", name); err == nil {
to.date = true
to.timeObj = t
names = append(names, formatDatePageName(to.timeObj), name)
} else {
names = append(names, name)
}
for _, name := range names {
blocks, err := loadBlocks(fp.dirname, name)
if err != nil && errors.Is(err, BlockNotFound) {
continue
}
return fp.blocksBackendGet(name, blocks, to)
}
page, err := fp.oldPagesBackend(name)
if err != nil {
return fp.blocksBackendGet(name, BlockResponse{ParentID: "root", PageID: name}, to)
}
return page
}
func (fp *FilePages) blocksBackendGet(name string, blocks BlockResponse, option titleOption) Page {
buf := bytes.Buffer{}
current := blocks.PageID
listItems := convertBlocksToListItems(current, blocks, 0)
if listItems == nil {
listItems = []ActualListItem{}
}
err := json.NewEncoder(&buf).Encode(&listItems)
if err != nil {
log.Printf("while encoding blocks: %s", err)
}
refs, err := getBackrefs(fp, name)
if err != nil {
refs = nil
}
title := formatTitle(blocks.Texts[name], option)
if title == "" {
title = cleanTitle(name)
}
return Page{
Name: name,
Title: title,
Content: buf.String(),
Refs: refs,
Blocks: blocks,
Parent: blocks.ParentID,
}
}
func formatTitle(title string, option titleOption) string {
if option.date {
return formatDateTitle(option.timeObj)
}
return title
}
func (fp *FilePages) oldPagesBackend(title string) (Page, error) {
name := strings.Replace(title, " ", "_", -1)
title = strings.Replace(title, "_", " ", -1)
refs, err := getBackrefs(fp, name)
if err != nil {
refs = nil
}
f, err := os.Open(filepath.Join(fp.dirname, name))
if err != nil {
log.Printf("while opening file in oldPagesBackend: %s", err)
return Page{
Title: title,
Name: name,
Content: "",
Refs: refs,
}, err
}
defer f.Close()
body, err := ioutil.ReadAll(f)
if err != nil {
log.Printf("while reading file in oldPagesBackend: %s", err)
return Page{
Name: name,
Title: title,
Content: "",
Refs: refs,
}, err
}
return Page{
Name: name,
Title: title,
Content: string(body),
Refs: refs,
}, nil
}
func (fp *FilePages) Save(p string, page Page, summary, author string) error {
fp.saveC <- saveMessage{p, page, summary, author}
return nil
}
func (fp *FilePages) save(msg saveMessage) error {
var sw stopwatch
p := msg.p
page := msg.page
summary := msg.summary
author := msg.author
page.Name = strings.Replace(p, " ", "_", -1)
page.Title = strings.Replace(p, "_", " ", -1)
sw.Start("create blocks")
err := saveBlocksFromPage(fp.dirname, page)
if err != nil {
log.Println(err)
}
sw.Stop()
if p[0] != '_' {
sw.Start("prepare")
f, err := os.Create(filepath.Join(fp.dirname, strings.Replace(p, " ", "_", -1)))
if err != nil {
return err
}
defer f.Close()
if page.Content[0] == '{' || page.Content[0] == '[' {
var buf bytes.Buffer
err = json.Indent(&buf, []byte(page.Content), "", " ")
if err != nil {
return err
}
_, err = buf.WriteTo(f)
if err != nil {
return err
}
} else {
f.WriteString(strings.Replace(page.Content, "\r\n", "\n", -1))
}
sw.Stop()
sw.Start("backrefs")
err = processBackrefs(fp.dirname, page)
if err != nil {
return fmt.Errorf("while processing backrefs: %s", err)
}
sw.Stop()
sw.Start("git")
err = saveWithGit(fp, p, summary, author)
if err != nil {
log.Printf("Error while saving to git: %w", err)
// return fmt.Errorf("while saving to git: %w", err)
}
sw.Stop()
sw.Start("index")
searchObjects, err := createSearchObjects(page.Name)
if err != nil {
return fmt.Errorf("while creating search object %s: %w", page.Name, err)
}
for _, so := range searchObjects {
if fp.index != nil {
err = fp.index.Index(so.ID, so)
if err != nil {
return fmt.Errorf("while indexing %s: %w", page.Name, err)
}
}
}
sw.Stop()
sw.Start("links")
err = saveLinksIncremental(fp.dirname, page.Title)
sw.Stop()
}
return nil
}
func saveWithNewIDs(dirname string, listItems []ListItemV2, pageName string) ([]ListItem, error) {
var newListItems []ListItem
for _, item := range listItems {
newItem := ListItem{
ID: item.ID.NewID(),
Indented: item.Indented,
Text: item.Text,
}
newListItems = append(newListItems, newItem)
}
return newListItems, nil
}
func saveBlocksFromPage(dirname string, page Page) error {
log.Println("Processing: ", page.Name)
var listItems []ListItem
err := json.NewDecoder(strings.NewReader(page.Content)).Decode(&listItems)
if err != nil {
log.Println("decoding default failed: %w", err)
var listItemsV2 []ListItemV2
err = json.NewDecoder(strings.NewReader(page.Content)).Decode(&listItemsV2)
listItems, err = saveWithNewIDs(dirname, listItemsV2, page.Name)
if err != nil {
return fmt.Errorf("while rewriting %s to use new ids: %w", page.Name, err)
}
}
blocks := make(map[string]Block)
prevList := make(map[string]ListItem)
root := "root"
parentBlock, err := loadBlock(dirname, page.Name)
if err != nil {
log.Println(err)
} else {
root = parentBlock.Parent
}
title := page.Title
if page.Name[0] == '_' {
title = parentBlock.Text
}
var parent = ListItem{
Text: title,
Indented: -1,
ID: page.Name,
}
prevList[parent.ID] = parent
blocks[parent.ID] = Block{
Text: title,
Children: []string{},
Parent: root,
}
var prev = &parent
for i, item := range listItems {
if item.Fleeting {
continue
}
prevList[item.ID] = item
if item.Indented > prev.Indented {
parent = *prev
} else if item.Indented == prev.Indented {
// nothing
} else if item.Indented <= parent.Indented {
for item.Indented <= parent.Indented {
if block, e := blocks[parent.ID]; e {
parent = prevList[block.Parent]
}
}
}
blocks[item.ID] = Block{item.Text, []string{}, parent.ID}
if block, e := blocks[parent.ID]; e {
block.Children = append(block.Children, item.ID)
blocks[parent.ID] = block
} else {
log.Println("Block missing")
}
prev = &listItems[i]
}
log.Printf("Loading parent block: %s", parent.ID)
f, err := os.Open(filepath.Join(dirname, BlocksDirectory, parent.ID))
if err == nil {
var parentBlock Block
err = json.NewDecoder(f).Decode(&parentBlock)
if err == nil {
if pb, e := blocks[parent.ID]; e {
pb.Text = parentBlock.Text
pb.Parent = parentBlock.Parent
blocks[parent.ID] = pb
log.Printf("Text=%s, Parent=%s", parentBlock.Text, parentBlock.Parent)
}
}
f.Close()
} else {
log.Println(err)
err = nil
}
for id, block := range blocks {
log.Println("Writing to ", id)
f, err := os.OpenFile(filepath.Join(dirname, BlocksDirectory, id), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
log.Println(err)
continue
}
enc := json.NewEncoder(f)
enc.SetIndent("", " ")
err = enc.Encode(&block)
if err != nil {
log.Println(err)
}
f.Close()
}
return err
}
func loadBlocks(dirname, rootBlockID string) (BlockResponse, error) {
resp := BlockResponse{
rootBlockID,
"",
nil,
nil,
nil,
}
resp.Texts = make(map[string]string)
resp.Children = make(map[string][]string)
queue := []string{rootBlockID}
block, err := loadBlock(dirname, rootBlockID)
if err != nil {
return BlockResponse{}, err
}
// NOTE: what does this do?
if rootBlockID[0] != '_' && block.Children == nil {
return BlockResponse{}, fmt.Errorf("not a block and has no children: %w", BlockNotFound)
}
prevID := rootBlockID
parentID := block.Parent
for parentID != "root" {
parent, err := loadBlock(dirname, parentID)
if err != nil {
return BlockResponse{}, fmt.Errorf("while loading current parent block (%s->%s): %w", prevID, parentID, err)
}
resp.Texts[parentID] = parent.Text
resp.Children[parentID] = parent.Children
resp.ParentID = parentID
resp.Parents = append(resp.Parents, parentID)
prevID = parentID
parentID = parent.Parent
}
if parentID == "root" {
resp.ParentID = "root"
}
for {
if len(queue) == 0 {
break
}
current := queue[0]
queue = queue[1:]
block, err := loadBlock(dirname, current)
if err != nil {
return BlockResponse{}, err
}
resp.Texts[current] = block.Text
resp.Children[current] = block.Children
queue = append(queue, block.Children...)
}
return resp, nil
}
func loadBlock(dirname, blockID string) (Block, error) {
f, err := os.Open(filepath.Join(dirname, BlocksDirectory, blockID))
if err != nil {
return Block{}, fmt.Errorf("%q: %w", blockID, BlockNotFound)
}
defer f.Close()
var block Block
err = json.NewDecoder(f).Decode(&block)
if err != nil {
return Block{}, fmt.Errorf("%q: %v", blockID, err)
}
return block, nil
}
func saveLinksIncremental(dirname, title string) error {
type Document struct {
Title string `json:"title"`
}
var results []Document
f, err := os.Open(filepath.Join(dirname, LinksFile))
if err != nil {
return err
}
err = json.NewDecoder(f).Decode(&results)
if err != nil {
return err
}
f.Close()
titles := make(map[string]bool)
for _, r := range results {
titles[r.Title] = true
}
// Add new? title
titles[title] = true
results = nil
for t, _ := range titles {
results = append(results, Document{t})
}
f, err = os.Create(filepath.Join(dirname, LinksFile))
err = json.NewEncoder(f).Encode(&results)
if err != nil {
return err
}
f.Close()
return nil
}
func saveLinks(mp PagesRepository) error {
type Document struct {
Title string `json:"title"`
}
var results []Document
pages, err := mp.AllPages()
if err != nil {
return err
}
for _, page := range pages {
results = append(results, Document{page.Title})
}
f, err := os.Create(filepath.Join(mp.(*FilePages).dirname, LinksFile))
if err != nil {
return err
}
defer f.Close()
err = json.NewEncoder(f).Encode(&results)
if err != nil {
return err
}
return nil
}
func saveWithGit(fp *FilePages, p string, summary, author string) error {
cmd := exec.Command("git", "add", ".")
cmd.Dir = fp.dirname
err := cmd.Run()
if err != nil {
return fmt.Errorf("while adding page %s: %s", p, err)
}
cmd = exec.Command("git", "commit", "-m", "Changes to "+p+" by "+author+"\n\n"+summary)
cmd.Dir = fp.dirname
// cmd.Stderr = os.Stderr
// cmd.Stdout = os.Stderr
err = cmd.Run()
if err != nil {
return fmt.Errorf("while commiting page %s: %s", p, err)
}
return nil
}
func (fp *FilePages) Exist(p string) bool {
f, err := os.Open(filepath.Join(fp.dirname, strings.Replace(p, " ", "_", -1)))
if err != nil {
return os.IsExist(err)
}
f.Close()
return true
}
func DiffPrettyHtml(diffs []diffmatchpatch.Diff) string {
var buff bytes.Buffer
for _, diff := range diffs {
// text := strings.Replace(html.EscapeString(diff.Text), "\n", "<span class=\"lighter\">&para;</span><br>", -1)
text := html.EscapeString(diff.Text)
switch diff.Type {
case diffmatchpatch.DiffInsert:
_, _ = buff.WriteString("<ins style=\"background:#e6ffe6;\">")
_, _ = buff.WriteString(text)
_, _ = buff.WriteString("</ins>")
case diffmatchpatch.DiffDelete:
_, _ = buff.WriteString("<del style=\"background:#ffe6e6;\">")
_, _ = buff.WriteString(text)
_, _ = buff.WriteString("</del>")
case diffmatchpatch.DiffEqual:
_, _ = buff.WriteString("<span>")
_, _ = buff.WriteString(text)
_, _ = buff.WriteString("</span>")
}
}
return buff.String()
}
func (fp *FilePages) PageHistory(p string) ([]Revision, error) {
page := strings.Replace(p, " ", "_", -1)
cmd := exec.Command("git", "log", "--pretty=oneline", "--no-decorate", "--color=never", page)
cmd.Dir = fp.dirname
output, err := cmd.StdoutPipe()
if err != nil {
return nil, err
}
defer output.Close()
err = cmd.Start()
if err != nil {
return nil, fmt.Errorf("while starting: %s", err)
}
buf := bufio.NewScanner(output)
var revisions []Revision
for buf.Scan() {
line := buf.Text()
start := strings.Index(line, " ")
commitId := line[0:start]
rest := line[start+1:]
pageText := gitRevision(fp.dirname, page, commitId)
revisions = append(revisions, Revision{
Version: commitId,
Page: DiffPage{Content: pageText},
Summary: rest,
})
}
dmp := diffmatchpatch.New()
prevText := ""
for i := len(revisions) - 1; i >= 0; i-- {
diffs := dmp.DiffMain(prevText, revisions[i].Page.Content, false)
revisions[i].Page.Diff = template.HTML(DiffPrettyHtml(diffs))
prevText = revisions[i].Page.Content
}
if err := cmd.Wait(); err != nil {
return nil, fmt.Errorf("while waiting: %s", err)
}
return revisions, nil
}
func gitRevision(dirname, page, version string) string {
cmd := exec.Command("git", "show", version+":"+page)
cmd.Dir = dirname
buf := bytes.Buffer{}
cmd.Stdout = &buf
cmd.Start()
cmd.Wait()
return buf.String()
}
func (fp *FilePages) RecentChanges() ([]Change, error) {
cmd := exec.Command("git", "log", "--format=--1--%nDate: %aI%n--2--%n%b%n--3--", "--name-only")
cmd.Dir = fp.dirname
buf := bytes.Buffer{}
cmd.Stdout = &buf
err := cmd.Start()
if err != nil {
return nil, err
}
err = cmd.Wait()
if err != nil {
return nil, err
}
scanner := bufio.NewScanner(&buf)
state := 0
var changes []Change
var change Change
body := ""
for scanner.Scan() {
line := scanner.Text()
if line == "--1--" {
state = 1
body = ""
continue
}
if line == "--2--" {
state = 2
continue
}
if line == "--3--" {
state = 3
continue
}
if state == 1 && strings.HasPrefix(line, "Date: ") {
line = line[6:]
changeTime, err := time.Parse(time.RFC3339, line)
if err != nil {
return changes, err
}
change.Date = changeTime
continue
}
if state == 2 {
if line == "" {
continue
}
body = body + line
continue
}
if state == 3 {
if line == "" {
continue
}
change.Page = line
}
change.Body = body
changes = append(changes, change)
}
return changes, nil
}
func (fp *FilePages) AllPages() ([]Page, error) {
log.Println("AllPages", fp.dirname)
files, err := ioutil.ReadDir(fp.dirname)
if err != nil {
return nil, err
}
var pages []Page
for _, file := range files {
if file.Name()[0] == '.' || file.Name()[0] == '_' {
continue
}
if file.Name() == "backrefs.json" {
continue
}
pages = append(pages, fp.Get(file.Name()))
}
return pages, nil
}