wiki/file.go

package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"html"
	"html/template"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/blevesearch/bleve"
	"github.com/sergi/go-diff/diffmatchpatch"
)

const (
	DocumentsFile   = "_documents.json"
	LinksFile       = "_links.json"
	BlocksDirectory = "_blocks"
)

type saveMessage struct {
	p       string
	page    Page
	summary string
	author  string
}

type Block struct {
	Text     string
	Children []string
	Parent   string
}

type ID struct {
	StrID  string
	WasInt bool
}

func (id *ID) UnmarshalJSON(data []byte) error {
	var intID int
	err := json.Unmarshal(data, &intID)
	if err == nil {
		*id = ID{strconv.FormatInt(int64(intID), 10), true}
		return nil
	}

	var strID string
	err = json.Unmarshal(data, &strID)
	if err == nil {
		*id = ID{strID, false}
		return nil
	}

	return fmt.Errorf("could not unmarshal %q as an int or string", data)
}

func (id *ID) NewID() string {
	if id.WasInt {
		l := time.Now().UnixNano()
		r := rand.Uint64()
		return fmt.Sprintf("_%d_%d", l, r)
	} else {
		return id.StrID
	}
}

// ListItemV2 is way to convert from old structure to new structure
type ListItemV2 struct {
	ID       ID
	Indented int
	Text     string
}

func (v2 ListItemV2) ListItem() ListItem {
	return ListItem{
		v2.ID.StrID,
		v2.Indented,
		v2.Text,
		false,
	}
}

// ListItem is a simplification of the information that was saved by the editor
type ListItem struct {
	ID       string
	Indented int
	Text     string
	Fleeting bool `json:"fleeting,omitempty"`
}

type ActualListItem struct {
	ID       string `json:"id"`
	Indented int    `json:"indented"`
	Text     string `json:"text"`
	Fold     string `json:"fold"`
	Hidden   bool   `json:"hidden"`
	Fleeting bool   `json:"fleeting,omitempty"`
}

type FilePages struct {
	dirname string
	saveC   chan saveMessage
	index   bleve.Index
}

func NewFilePages(dirname string, index bleve.Index) PagesRepository {
	err := os.MkdirAll(filepath.Join(dirname, "_blocks"), 0777)
	if err != nil {
		log.Fatalln(err)
	}

	fp := &FilePages{dirname, make(chan saveMessage), index}

	go func() {
		for msg := range fp.saveC {
			err := fp.save(msg)
			if err != nil {
				log.Println(err)
			}
		}
	}()
	return fp
}

func convertBlocksToListItems(current string, blocks BlockResponse, indent int) []ActualListItem {
	listItems := []ActualListItem{}

	for _, child := range blocks.Children[current] {
		l := convertBlocksToListItems(child, blocks, indent+1)
		listItems = append(listItems,
			ActualListItem{
				ID:       child,
				Indented: indent,
				Text:     blocks.Texts[child],
				Fold:     "open", // TODO: keep Fold state somewhere
				Hidden:   false,
			})
		listItems = append(listItems, l...)
	}

	return listItems
}

func (fp *FilePages) Get(title string) Page {
	var sw stopwatch
	sw.Start("Get " + title)
	defer sw.Stop()

	refs, err := getBackrefs(fp, title)
	if err != nil {
		refs = nil
	}

	blocks, err := loadBlocks(fp.dirname, title)
	if err != nil {
		return fp.oldPagesBackend(title, blocks, refs)
	}

	// Blocks based pages
	return fp.blocksBackendGet(title, blocks, refs)
}

func (fp *FilePages) blocksBackendGet(name string, blocks BlockResponse, refs map[string][]Backref) Page {
	buf := bytes.Buffer{}
	current := blocks.PageID
	listItems := convertBlocksToListItems(current, blocks, 0)
	if listItems == nil {
		listItems = []ActualListItem{}
	}

	err := json.NewEncoder(&buf).Encode(&listItems)
	if err != nil {
		log.Printf("while encoding blocks: %s", err)
	}

	return Page{
		Name:    name,
		Title:   blocks.Texts[name],
		Content: buf.String(),
		Refs:    refs,
		Blocks:  blocks,
		Parent:  blocks.ParentID,
	}
}

func (fp *FilePages) oldPagesBackend(title string, blocks BlockResponse, refs map[string][]Backref) Page {
	name := strings.Replace(title, " ", "_", -1)
	title = strings.Replace(title, "_", " ", -1)

	f, err := os.Open(filepath.Join(fp.dirname, name))
	if err != nil {
		log.Printf("while opening file in oldPagesBackend: %s", err)
		return Page{
			Title:   title,
			Name:    name,
			Content: "",
			Refs:    refs,
			Blocks:  blocks,
		}
	}

	defer f.Close()
	body, err := ioutil.ReadAll(f)
	if err != nil {
		log.Printf("while reading file in oldPagesBackend: %s", err)
		return Page{
			Name:    name,
			Title:   title,
			Content: "",
			Refs:    refs,
			Blocks:  blocks,
		}
	}

	return Page{
		Name:    name,
		Title:   title,
		Content: string(body),
		Refs:    refs,
		Blocks:  blocks,
	}
}

func (fp *FilePages) Save(p string, page Page, summary, author string) error {
	fp.saveC <- saveMessage{p, page, summary, author}
	return nil
}

func (fp *FilePages) save(msg saveMessage) error {
	var sw stopwatch
	p := msg.p
	page := msg.page
	summary := msg.summary
	author := msg.author

	page.Name = strings.Replace(p, " ", "_", -1)
	page.Title = strings.Replace(p, "_", " ", -1)

	sw.Start("create blocks")
	err := saveBlocksFromPage(fp.dirname, page)
	if err != nil {
		log.Println(err)
	}
	sw.Stop()

	if p[0] != '_' {
		sw.Start("prepare")

		f, err := os.Create(filepath.Join(fp.dirname, strings.Replace(p, " ", "_", -1)))
		if err != nil {
			return err
		}
		defer f.Close()
		if page.Content[0] == '{' || page.Content[0] == '[' {
			var buf bytes.Buffer
			err = json.Indent(&buf, []byte(page.Content), "", "  ")
			if err != nil {
				return err
			}
			_, err = buf.WriteTo(f)
			if err != nil {
				return err
			}
		} else {
			f.WriteString(strings.Replace(page.Content, "\r\n", "\n", -1))
		}
		sw.Stop()
		sw.Start("backrefs")
		err = processBackrefs(fp.dirname, page)
		if err != nil {
			return fmt.Errorf("while processing backrefs: %s", err)
		}
		sw.Stop()

		sw.Start("git")
		err = saveWithGit(fp, p, summary, author)
		if err != nil {
			log.Printf("Error while saving to git: %w", err)
			// return fmt.Errorf("while saving to git: %w", err)
		}
		sw.Stop()

		sw.Start("index")
		searchObjects, err := createSearchObjects(page.Name)
		if err != nil {
			return fmt.Errorf("while creating search object %s: %w", page.Name, err)
		}
		for _, so := range searchObjects {
			if fp.index != nil {
				err = fp.index.Index(so.ID, so)
				if err != nil {
					return fmt.Errorf("while indexing %s: %w", page.Name, err)
				}
			}
		}
		sw.Stop()
		sw.Start("links")
		err = saveLinksIncremental(fp.dirname, page.Title)
		sw.Stop()
	}

	return nil
}

func saveWithNewIDs(dirname string, listItems []ListItemV2, pageName string) ([]ListItem, error) {
	var newListItems []ListItem
	for _, item := range listItems {
		newItem := ListItem{
			ID:       item.ID.NewID(),
			Indented: item.Indented,
			Text:     item.Text,
		}
		newListItems = append(newListItems, newItem)
	}

	return newListItems, nil
}

func saveBlocksFromPage(dirname string, page Page) error {
	log.Println("Processing: ", page.Name)
	var listItems []ListItem
	err := json.NewDecoder(strings.NewReader(page.Content)).Decode(&listItems)
	if err != nil {
		log.Println("decoding default failed: %w", err)
		var listItemsV2 []ListItemV2
		err = json.NewDecoder(strings.NewReader(page.Content)).Decode(&listItemsV2)
		listItems, err = saveWithNewIDs(dirname, listItemsV2, page.Name)
		if err != nil {
			return fmt.Errorf("while rewriting %s to use new ids: %w", page.Name, err)
		}
	}

	blocks := make(map[string]Block)
	prevList := make(map[string]ListItem)

	root := "root"
	parentBlock, err := loadBlock(dirname, page.Name)
	if err != nil {
		log.Println(err)
	} else {
		root = parentBlock.Parent
	}

	title := page.Title
	if page.Name[0] == '_' {
		title = parentBlock.Text
	}

	var parent = ListItem{
		Text:     title,
		Indented: -1,
		ID:       page.Name,
	}

	prevList[parent.ID] = parent
	blocks[parent.ID] = Block{
		Text:     title,
		Children: []string{},
		Parent:   root,
	}

	var prev = &parent

	for i, item := range listItems {
		if item.Fleeting {
			continue
		}

		prevList[item.ID] = item
		if item.Indented > prev.Indented {
			parent = *prev
		} else if item.Indented == prev.Indented {
			// nothing
		} else if item.Indented <= parent.Indented {
			for item.Indented <= parent.Indented {
				if block, e := blocks[parent.ID]; e {
					parent = prevList[block.Parent]
				}
			}
		}

		blocks[item.ID] = Block{item.Text, []string{}, parent.ID}
		if block, e := blocks[parent.ID]; e {
			block.Children = append(block.Children, item.ID)
			blocks[parent.ID] = block
		} else {
			log.Println("Block missing")
		}
		prev = &listItems[i]
	}

	log.Printf("Loading parent block: %s", parent.ID)
	f, err := os.Open(filepath.Join(dirname, BlocksDirectory, parent.ID))
	if err == nil {
		var parentBlock Block
		err = json.NewDecoder(f).Decode(&parentBlock)
		if err == nil {
			if pb, e := blocks[parent.ID]; e {
				pb.Text = parentBlock.Text
				pb.Parent = parentBlock.Parent
				blocks[parent.ID] = pb
				log.Printf("Text=%s, Parent=%s", parentBlock.Text, parentBlock.Parent)
			}
		}
		f.Close()
	} else {
		log.Println(err)
		err = nil
	}

	for id, block := range blocks {
		log.Println("Writing to ", id)
		f, err := os.OpenFile(filepath.Join(dirname, BlocksDirectory, id), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
		if err != nil {
			log.Println(err)
			continue
		}
		enc := json.NewEncoder(f)
		enc.SetIndent("", "  ")
		err = enc.Encode(&block)
		if err != nil {
			log.Println(err)
		}
		f.Close()
	}

	return err
}

type BlockResponse struct {
	PageID   string
	ParentID string
	Texts    map[string]string
	Children map[string][]string
	Parents  []string
}

func loadBlocks(dirname, rootBlockID string) (BlockResponse, error) {
	resp := BlockResponse{
		rootBlockID,
		"",
		nil,
		nil,
		nil,
	}

	resp.Texts = make(map[string]string)
	resp.Children = make(map[string][]string)

	queue := []string{rootBlockID}

	block, err := loadBlock(dirname, rootBlockID)
	if err != nil {
		return BlockResponse{}, fmt.Errorf("while loading current block (%s): %w", rootBlockID, err)
	}
	if rootBlockID[0] != '_' && block.Children == nil {
		return BlockResponse{}, fmt.Errorf("while loading current block (%s): not a block and no children", rootBlockID)
	}

	prevID := rootBlockID
	parentID := block.Parent

	for parentID != "root" {
		parent, err := loadBlock(dirname, parentID)
		if err != nil {
			return BlockResponse{}, fmt.Errorf("while loading current parent block (%s->%s): %w", prevID, parentID, err)
		}

		resp.Texts[parentID] = parent.Text
		resp.Children[parentID] = parent.Children
		resp.ParentID = parentID

		resp.Parents = append(resp.Parents, parentID)

		prevID = parentID
		parentID = parent.Parent
	}
	if parentID == "root" {
		resp.ParentID = "root"
	}

	for {
		if len(queue) == 0 {
			break
		}
		current := queue[0]
		queue = queue[1:]
		block, err := loadBlock(dirname, current)
		if err != nil {
			return BlockResponse{}, fmt.Errorf("while loading block (%s): %w", current, err)
		}
		resp.Texts[current] = block.Text
		resp.Children[current] = block.Children
		queue = append(queue, block.Children...)
	}

	return resp, nil
}

func loadBlock(dirname, blockID string) (Block, error) {
	f, err := os.Open(filepath.Join(dirname, BlocksDirectory, blockID))
	if err != nil {
		return Block{}, err
	}
	defer f.Close()
	var block Block
	err = json.NewDecoder(f).Decode(&block)
	if err != nil {
		return Block{}, err
	}
	return block, nil
}

func saveLinksIncremental(dirname, title string) error {
	type Document struct {
		Title string `json:"title"`
	}
	var results []Document
	f, err := os.Open(filepath.Join(dirname, LinksFile))
	if err != nil {
		return err
	}
	err = json.NewDecoder(f).Decode(&results)
	if err != nil {
		return err
	}
	f.Close()

	titles := make(map[string]bool)
	for _, r := range results {
		titles[r.Title] = true
	}

	// Add new? title
	titles[title] = true

	results = nil
	for t, _ := range titles {
		results = append(results, Document{t})
	}

	f, err = os.Create(filepath.Join(dirname, LinksFile))
	err = json.NewEncoder(f).Encode(&results)
	if err != nil {
		return err
	}
	f.Close()

	return nil
}

func saveLinks(mp PagesRepository) error {
	type Document struct {
		Title string `json:"title"`
	}
	var results []Document
	pages, err := mp.AllPages()
	if err != nil {
		return err
	}
	for _, page := range pages {
		results = append(results, Document{page.Title})
	}
	f, err := os.Create(filepath.Join(mp.(*FilePages).dirname, LinksFile))
	if err != nil {
		return err
	}
	defer f.Close()
	err = json.NewEncoder(f).Encode(&results)
	if err != nil {
		return err
	}
	return nil
}

func saveWithGit(fp *FilePages, p string, summary, author string) error {
	cmd := exec.Command("git", "add", ".")
	cmd.Dir = fp.dirname
	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("while adding page %s: %s", p, err)
	}

	cmd = exec.Command("git", "commit", "-m", "Changes to "+p+" by "+author+"\n\n"+summary)
	cmd.Dir = fp.dirname
	// cmd.Stderr = os.Stderr
	// cmd.Stdout = os.Stderr
	err = cmd.Run()
	if err != nil {
		return fmt.Errorf("while commiting page %s: %s", p, err)
	}
	return nil
}

func (fp *FilePages) Exist(p string) bool {
	f, err := os.Open(filepath.Join(fp.dirname, strings.Replace(p, " ", "_", -1)))
	if err != nil {
		return os.IsExist(err)
	}
	f.Close()
	return true
}

func DiffPrettyHtml(diffs []diffmatchpatch.Diff) string {
	var buff bytes.Buffer
	for _, diff := range diffs {
		// text := strings.Replace(html.EscapeString(diff.Text), "\n", "<span class=\"lighter\">&para;</span><br>", -1)
		text := html.EscapeString(diff.Text)
		switch diff.Type {
		case diffmatchpatch.DiffInsert:
			_, _ = buff.WriteString("<ins style=\"background:#e6ffe6;\">")
			_, _ = buff.WriteString(text)
			_, _ = buff.WriteString("</ins>")
		case diffmatchpatch.DiffDelete:
			_, _ = buff.WriteString("<del style=\"background:#ffe6e6;\">")
			_, _ = buff.WriteString(text)
			_, _ = buff.WriteString("</del>")
		case diffmatchpatch.DiffEqual:
			_, _ = buff.WriteString("<span>")
			_, _ = buff.WriteString(text)
			_, _ = buff.WriteString("</span>")
		}
	}
	return buff.String()
}

func (fp *FilePages) PageHistory(p string) ([]Revision, error) {
	page := strings.Replace(p, " ", "_", -1)
	cmd := exec.Command("git", "log", "--pretty=oneline", "--no-decorate", "--color=never", page)
	cmd.Dir = fp.dirname
	output, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	defer output.Close()

	err = cmd.Start()
	if err != nil {
		return nil, fmt.Errorf("while starting: %s", err)
	}

	buf := bufio.NewScanner(output)

	var revisions []Revision

	for buf.Scan() {
		line := buf.Text()
		start := strings.Index(line, " ")
		commitId := line[0:start]
		rest := line[start+1:]
		pageText := gitRevision(fp.dirname, page, commitId)
		revisions = append(revisions, Revision{
			Version: commitId,
			Page:    DiffPage{Content: pageText},
			Summary: rest,
		})
	}

	dmp := diffmatchpatch.New()
	prevText := ""

	for i := len(revisions) - 1; i >= 0; i-- {
		diffs := dmp.DiffMain(prevText, revisions[i].Page.Content, false)
		revisions[i].Page.Diff = template.HTML(DiffPrettyHtml(diffs))
		prevText = revisions[i].Page.Content
	}

	if err := cmd.Wait(); err != nil {
		return nil, fmt.Errorf("while waiting: %s", err)
	}

	return revisions, nil
}

func gitRevision(dirname, page, version string) string {
	cmd := exec.Command("git", "show", version+":"+page)
	cmd.Dir = dirname
	buf := bytes.Buffer{}
	cmd.Stdout = &buf
	cmd.Start()
	cmd.Wait()
	return buf.String()
}

func (fp *FilePages) RecentChanges() ([]Change, error) {
	cmd := exec.Command("git", "log", "--format=--1--%nDate: %aI%n--2--%n%b%n--3--", "--name-only")
	cmd.Dir = fp.dirname
	buf := bytes.Buffer{}
	cmd.Stdout = &buf
	err := cmd.Start()
	if err != nil {
		return nil, err
	}
	err = cmd.Wait()
	if err != nil {
		return nil, err
	}

	scanner := bufio.NewScanner(&buf)

	state := 0

	var changes []Change
	var change Change

	body := ""

	for scanner.Scan() {
		line := scanner.Text()

		if line == "--1--" {
			state = 1

			body = ""
			continue
		}

		if line == "--2--" {
			state = 2
			continue
		}

		if line == "--3--" {
			state = 3
			continue
		}

		if state == 1 && strings.HasPrefix(line, "Date: ") {
			line = line[6:]
			changeTime, err := time.Parse(time.RFC3339, line)
			if err != nil {
				return changes, err
			}
			change.Date = changeTime
			continue
		}

		if state == 2 {
			if line == "" {
				continue
			}
			body = body + line
			continue
		}

		if state == 3 {
			if line == "" {
				continue
			}
			change.Page = line
		}

		change.Body = body
		changes = append(changes, change)
	}

	return changes, nil
}

func (fp *FilePages) AllPages() ([]Page, error) {
	log.Println("AllPages", fp.dirname)

	files, err := ioutil.ReadDir(fp.dirname)
	if err != nil {
		return nil, err
	}

	var pages []Page
	for _, file := range files {
		if file.Name()[0] == '.' || file.Name()[0] == '_' {
			continue
		}
		if file.Name() == "backrefs.json" {
			continue
		}
		pages = append(pages, fp.Get(file.Name()))
	}

	return pages, nil
}