wiki/file.go

package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"html"
	"html/template"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/blevesearch/bleve"
	"github.com/sergi/go-diff/diffmatchpatch"
)

const (
	DocumentsFile = "_documents.json"
	LinksFile     = "_links.json"
)

type saveMessage struct {
	p       string
	page    Page
	summary string
	author  string
}

type FilePages struct {
	dirname string
	saveC   chan saveMessage
	index   bleve.Index
}

func NewFilePages(dirname string, index bleve.Index) PagesRepository {
	fp := &FilePages{dirname, make(chan saveMessage), index}
	go func() {
		for msg := range fp.saveC {
			err := fp.save(msg)
			if err != nil {
				log.Println(err)
			}
		}
	}()
	return fp
}

func (fp *FilePages) Get(title string) Page {
	name := strings.Replace(title, " ", "_", -1)
	title = strings.Replace(title, "_", " ", -1)
	refs, err := getBackrefs(fp, name)
	if err != nil {
		refs = nil
	}

	f, err := os.Open(filepath.Join(fp.dirname, name))
	if err != nil {
		return Page{
			Title:   title,
			Name:    name,
			Content: "",
			Refs:    refs,
		}
	}
	defer f.Close()
	body, err := ioutil.ReadAll(f)
	if err != nil {
		return Page{
			Title:   title,
			Name:    name,
			Content: "",
			Refs:    refs,
		}
	}
	return Page{
		Name:    name,
		Title:   title,
		Content: string(body),
		Refs:    refs,
	}
}

func (fp *FilePages) Save(p string, page Page, summary, author string) error {
	fp.saveC <- saveMessage{p, page, summary, author}
	return nil
}

func (fp *FilePages) save(msg saveMessage) error {
	var sw stopwatch
	sw.Start("prepare")
	p := msg.p
	page := msg.page
	summary := msg.summary
	author := msg.author

	page.Name = strings.Replace(p, " ", "_", -1)
	page.Title = strings.Replace(p, "_", " ", -1)

	f, err := os.Create(filepath.Join(fp.dirname, strings.Replace(p, " ", "_", -1)))
	if err != nil {
		return err
	}
	defer f.Close()
	if page.Content[0] == '{' || page.Content[0] == '[' {
		var buf bytes.Buffer
		err = json.Indent(&buf, []byte(page.Content), "", "  ")
		if err != nil {
			return err
		}
		_, err = buf.WriteTo(f)
		if err != nil {
			return err
		}
	} else {
		f.WriteString(strings.Replace(page.Content, "\r\n", "\n", -1))
	}
	sw.Stop()
	sw.Start("backrefs")
	err = processBackrefs(fp.dirname, page)
	if err != nil {
		return fmt.Errorf("while processing backrefs: %s", err)
	}
	sw.Stop()

	sw.Start("git")
	err = saveWithGit(fp, p, summary, author)
	if err != nil {
		return fmt.Errorf("while saving to git: %w", err)
	}
	sw.Stop()
	sw.Start("index")
	so, err := createSearchObject(page)
	if err != nil {
		return fmt.Errorf("while creating search object %s: %w", page.Name, err)
	}
	err = fp.index.Index(page.Name, so)
	if err != nil {
		return fmt.Errorf("while indexing %s: %w", page.Name, err)
	}
	sw.Stop()
	sw.Start("links")
	err = saveLinksIncremental(fp.dirname, page.Title)
	sw.Stop()
	return err
}

func saveLinksIncremental(dirname, title string) error {
	type Document struct {
		Title string `json:"title"`
	}
	var results []Document
	f, err := os.Open(filepath.Join(dirname, LinksFile))
	if err != nil {
		return err
	}
	err = json.NewDecoder(f).Decode(&results)
	if err != nil {
		return err
	}
	f.Close()

	titles := make(map[string]bool)
	for _, r := range results {
		titles[r.Title] = true
	}

	// Add new? title
	titles[title] = true

	results = nil
	for t, _ := range titles {
		results = append(results, Document{t})
	}

	f, err = os.Create(filepath.Join(dirname, LinksFile))
	err = json.NewEncoder(f).Encode(&results)
	if err != nil {
		return err
	}
	f.Close()

	return nil
}

func saveLinks(fp *FilePages) error {
	type Document struct {
		Title string `json:"title"`
	}
	var results []Document
	pages, err := mp.(*FilePages).AllPages()
	if err != nil {
		return err
	}
	for _, page := range pages {
		results = append(results, Document{page.Title})
	}
	f, err := os.Create(filepath.Join(fp.dirname, LinksFile))
	if err != nil {
		return err
	}
	defer f.Close()
	err = json.NewEncoder(f).Encode(&results)
	if err != nil {
		return err
	}
	return nil
}

func saveDocuments(fp *FilePages) error {
	type Document struct {
		Title string `json:"title"`
		Body  string `json:"body"`
		URL   string `json:"url"`
	}
	var results []Document
	pages, err := mp.(*FilePages).AllPages()
	if err != nil {
		return err
	}
	for _, page := range pages {
		content := strings.Builder{}

		var listItems []struct {
			Indented int
			Text     string
		}

		err = json.NewDecoder(strings.NewReader(page.Content)).Decode(&listItems)
		if err == nil {
			for _, item := range listItems {
				content.WriteString(item.Text)
				content.WriteByte(' ')
			}
		} else {
			content.WriteString(page.Content)
			content.WriteByte(' ')
		}

		for page, refs := range page.Refs {
			content.WriteString(page)
			content.WriteByte(' ')
			for _, ref := range refs {
				content.WriteString(ref.Line)
				content.WriteByte(' ')
			}
		}

		results = append(results, Document{
			Title: page.Title,
			Body:  content.String(),
			URL:   page.Name,
		})
	}

	f, err := os.Create(filepath.Join(fp.dirname, DocumentsFile))
	if err != nil {
		return err
	}
	defer f.Close()
	err = json.NewEncoder(f).Encode(&results)
	if err != nil {
		return err
	}

	return nil
}

func saveWithGit(fp *FilePages, p string, summary, author string) error {
	cmd := exec.Command("git", "add", ".")
	cmd.Dir = fp.dirname
	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("while adding page %s: %s", p, err)
	}

	cmd = exec.Command("git", "commit", "-m", "Changes to "+p+" by "+author+"\n\n"+summary)
	cmd.Dir = fp.dirname
	err = cmd.Run()
	if err != nil {
		return fmt.Errorf("while commiting page %s: %s", p, err)
	}
	return nil
}

func (fp *FilePages) Exist(p string) bool {
	f, err := os.Open(filepath.Join(fp.dirname, strings.Replace(p, " ", "_", -1)))
	if err != nil {
		return os.IsExist(err)
	}
	f.Close()
	return true
}

func DiffPrettyHtml(diffs []diffmatchpatch.Diff) string {
	var buff bytes.Buffer
	for _, diff := range diffs {
		// text := strings.Replace(html.EscapeString(diff.Text), "\n", "<span class=\"lighter\">&para;</span><br>", -1)
		text := html.EscapeString(diff.Text)
		switch diff.Type {
		case diffmatchpatch.DiffInsert:
			_, _ = buff.WriteString("<ins style=\"background:#e6ffe6;\">")
			_, _ = buff.WriteString(text)
			_, _ = buff.WriteString("</ins>")
		case diffmatchpatch.DiffDelete:
			_, _ = buff.WriteString("<del style=\"background:#ffe6e6;\">")
			_, _ = buff.WriteString(text)
			_, _ = buff.WriteString("</del>")
		case diffmatchpatch.DiffEqual:
			_, _ = buff.WriteString("<span>")
			_, _ = buff.WriteString(text)
			_, _ = buff.WriteString("</span>")
		}
	}
	return buff.String()
}

func (fp *FilePages) PageHistory(p string) ([]Revision, error) {
	page := strings.Replace(p, " ", "_", -1)
	cmd := exec.Command("git", "log", "--pretty=oneline", "--no-decorate", "--color=never", page)
	cmd.Dir = fp.dirname
	output, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	defer output.Close()

	err = cmd.Start()
	if err != nil {
		return nil, fmt.Errorf("while starting: %s", err)
	}

	buf := bufio.NewScanner(output)

	var revisions []Revision

	for buf.Scan() {
		line := buf.Text()
		start := strings.Index(line, " ")
		commitId := line[0:start]
		rest := line[start+1:]
		pageText := gitRevision(fp.dirname, page, commitId)
		revisions = append(revisions, Revision{
			Version: commitId,
			Page:    DiffPage{Content: pageText},
			Summary: rest,
		})
	}

	dmp := diffmatchpatch.New()
	prevText := ""

	for i := len(revisions) - 1; i >= 0; i-- {
		diffs := dmp.DiffMain(prevText, revisions[i].Page.Content, false)
		revisions[i].Page.Diff = template.HTML(DiffPrettyHtml(diffs))
		prevText = revisions[i].Page.Content
	}

	if err := cmd.Wait(); err != nil {
		return nil, fmt.Errorf("while waiting: %s", err)
	}

	return revisions, nil
}

func gitRevision(dirname, page, version string) string {
	cmd := exec.Command("git", "show", version+":"+page)
	cmd.Dir = dirname
	buf := bytes.Buffer{}
	cmd.Stdout = &buf
	cmd.Start()
	cmd.Wait()
	return buf.String()
}

func (fp *FilePages) RecentChanges() ([]Change, error) {
	cmd := exec.Command("git", "log", "--format=--1--%nDate: %aI%n--2--%n%b%n--3--", "--name-only")
	cmd.Dir = fp.dirname
	buf := bytes.Buffer{}
	cmd.Stdout = &buf
	err := cmd.Start()
	if err != nil {
		return nil, err
	}
	err = cmd.Wait()
	if err != nil {
		return nil, err
	}

	scanner := bufio.NewScanner(&buf)

	state := 0

	var changes []Change
	var change Change

	body := ""

	for scanner.Scan() {
		line := scanner.Text()

		if line == "--1--" {
			state = 1

			body = ""
			continue
		}

		if line == "--2--" {
			state = 2
			continue
		}

		if line == "--3--" {
			state = 3
			continue
		}

		if state == 1 && strings.HasPrefix(line, "Date: ") {
			line = line[6:]
			changeTime, err := time.Parse(time.RFC3339, line)
			if err != nil {
				return changes, err
			}
			change.Date = changeTime
			continue
		}

		if state == 2 {
			if line == "" {
				continue
			}
			body = body + line
			continue
		}

		if state == 3 {
			if line == "" {
				continue
			}
			change.Page = line
		}

		change.Body = body
		changes = append(changes, change)
	}

	return changes, nil
}

func (fp *FilePages) AllPages() ([]Page, error) {
	files, err := ioutil.ReadDir(fp.dirname)
	if err != nil {
		return nil, err
	}

	var pages []Page
	for _, file := range files {
		if file.Name()[0] == '.' || file.Name()[0] == '_' {
			continue
		}
		if file.Name() == "backrefs.json" {
			continue
		}
		pages = append(pages, fp.Get(file.Name()))
	}

	return pages, nil
}