package main import ( "bufio" "bytes" "encoding/json" "fmt" "html" "html/template" "io/ioutil" "log" "os" "os/exec" "path/filepath" "strings" "time" "github.com/blevesearch/bleve" "github.com/sergi/go-diff/diffmatchpatch" ) const ( DocumentsFile = "_documents.json" LinksFile = "_links.json" ) type saveMessage struct { p string page Page summary string author string } type FilePages struct { dirname string saveC chan saveMessage index bleve.Index } func NewFilePages(dirname string, index bleve.Index) PagesRepository { fp := &FilePages{dirname, make(chan saveMessage), index} go func() { for msg := range fp.saveC { err := fp.save(msg) if err != nil { log.Println(err) } } }() return fp } func (fp *FilePages) Get(title string) Page { name := strings.Replace(title, " ", "_", -1) title = strings.Replace(title, "_", " ", -1) refs, err := getBackrefs(fp, name) if err != nil { refs = nil } f, err := os.Open(filepath.Join(fp.dirname, name)) if err != nil { return Page{ Title: title, Name: name, Content: "", Refs: refs, } } defer f.Close() body, err := ioutil.ReadAll(f) if err != nil { return Page{ Title: title, Name: name, Content: "", Refs: refs, } } return Page{ Name: name, Title: title, Content: string(body), Refs: refs, } } func (fp *FilePages) Save(p string, page Page, summary, author string) error { fp.saveC <- saveMessage{p, page, summary, author} return nil } func (fp *FilePages) save(msg saveMessage) error { var sw stopwatch sw.Start("prepare") p := msg.p page := msg.page summary := msg.summary author := msg.author page.Name = strings.Replace(p, " ", "_", -1) page.Title = strings.Replace(p, "_", " ", -1) f, err := os.Create(filepath.Join(fp.dirname, strings.Replace(p, " ", "_", -1))) if err != nil { return err } defer f.Close() if page.Content[0] == '{' || page.Content[0] == '[' { var buf bytes.Buffer err = json.Indent(&buf, []byte(page.Content), "", " ") if err != nil { return err } _, err = buf.WriteTo(f) if err != nil { return err } } else { f.WriteString(strings.Replace(page.Content, "\r\n", "\n", -1)) } sw.Stop() sw.Start("backrefs") err = processBackrefs(fp.dirname, page) if err != nil { return fmt.Errorf("while processing backrefs: %s", err) } sw.Stop() sw.Start("git") err = saveWithGit(fp, p, summary, author) if err != nil { return fmt.Errorf("while saving to git: %w", err) } sw.Stop() sw.Start("index") so, err := createSearchObject(page) if err != nil { return fmt.Errorf("while creating search object %s: %w", page.Name, err) } if fp.index != nil { err = fp.index.Index(page.Name, so) if err != nil { return fmt.Errorf("while indexing %s: %w", page.Name, err) } } sw.Stop() sw.Start("links") err = saveLinksIncremental(fp.dirname, page.Title) sw.Stop() return err } func saveLinksIncremental(dirname, title string) error { type Document struct { Title string `json:"title"` } var results []Document f, err := os.Open(filepath.Join(dirname, LinksFile)) if err != nil { return err } err = json.NewDecoder(f).Decode(&results) if err != nil { return err } f.Close() titles := make(map[string]bool) for _, r := range results { titles[r.Title] = true } // Add new? title titles[title] = true results = nil for t, _ := range titles { results = append(results, Document{t}) } f, err = os.Create(filepath.Join(dirname, LinksFile)) err = json.NewEncoder(f).Encode(&results) if err != nil { return err } f.Close() return nil } func saveLinks(fp *FilePages) error { type Document struct { Title string `json:"title"` } var results []Document pages, err := mp.(*FilePages).AllPages() if err != nil { return err } for _, page := range pages { results = append(results, Document{page.Title}) } f, err := os.Create(filepath.Join(fp.dirname, LinksFile)) if err != nil { return err } defer f.Close() err = json.NewEncoder(f).Encode(&results) if err != nil { return err } return nil } func saveDocuments(fp *FilePages) error { type Document struct { Title string `json:"title"` Body string `json:"body"` URL string `json:"url"` } var results []Document pages, err := mp.(*FilePages).AllPages() if err != nil { return err } for _, page := range pages { content := strings.Builder{} var listItems []struct { Indented int Text string } err = json.NewDecoder(strings.NewReader(page.Content)).Decode(&listItems) if err == nil { for _, item := range listItems { content.WriteString(item.Text) content.WriteByte(' ') } } else { content.WriteString(page.Content) content.WriteByte(' ') } for page, refs := range page.Refs { content.WriteString(page) content.WriteByte(' ') for _, ref := range refs { content.WriteString(ref.Line) content.WriteByte(' ') } } results = append(results, Document{ Title: page.Title, Body: content.String(), URL: page.Name, }) } f, err := os.Create(filepath.Join(fp.dirname, DocumentsFile)) if err != nil { return err } defer f.Close() err = json.NewEncoder(f).Encode(&results) if err != nil { return err } return nil } func saveWithGit(fp *FilePages, p string, summary, author string) error { cmd := exec.Command("git", "add", ".") cmd.Dir = fp.dirname err := cmd.Run() if err != nil { return fmt.Errorf("while adding page %s: %s", p, err) } cmd = exec.Command("git", "commit", "-m", "Changes to "+p+" by "+author+"\n\n"+summary) cmd.Dir = fp.dirname err = cmd.Run() if err != nil { return fmt.Errorf("while commiting page %s: %s", p, err) } return nil } func (fp *FilePages) Exist(p string) bool { f, err := os.Open(filepath.Join(fp.dirname, strings.Replace(p, " ", "_", -1))) if err != nil { return os.IsExist(err) } f.Close() return true } func DiffPrettyHtml(diffs []diffmatchpatch.Diff) string { var buff bytes.Buffer for _, diff := range diffs { // text := strings.Replace(html.EscapeString(diff.Text), "\n", "
", -1) text := html.EscapeString(diff.Text) switch diff.Type { case diffmatchpatch.DiffInsert: _, _ = buff.WriteString("") _, _ = buff.WriteString(text) _, _ = buff.WriteString("") case diffmatchpatch.DiffDelete: _, _ = buff.WriteString("") _, _ = buff.WriteString(text) _, _ = buff.WriteString("") case diffmatchpatch.DiffEqual: _, _ = buff.WriteString("") _, _ = buff.WriteString(text) _, _ = buff.WriteString("") } } return buff.String() } func (fp *FilePages) PageHistory(p string) ([]Revision, error) { page := strings.Replace(p, " ", "_", -1) cmd := exec.Command("git", "log", "--pretty=oneline", "--no-decorate", "--color=never", page) cmd.Dir = fp.dirname output, err := cmd.StdoutPipe() if err != nil { return nil, err } defer output.Close() err = cmd.Start() if err != nil { return nil, fmt.Errorf("while starting: %s", err) } buf := bufio.NewScanner(output) var revisions []Revision for buf.Scan() { line := buf.Text() start := strings.Index(line, " ") commitId := line[0:start] rest := line[start+1:] pageText := gitRevision(fp.dirname, page, commitId) revisions = append(revisions, Revision{ Version: commitId, Page: DiffPage{Content: pageText}, Summary: rest, }) } dmp := diffmatchpatch.New() prevText := "" for i := len(revisions) - 1; i >= 0; i-- { diffs := dmp.DiffMain(prevText, revisions[i].Page.Content, false) revisions[i].Page.Diff = template.HTML(DiffPrettyHtml(diffs)) prevText = revisions[i].Page.Content } if err := cmd.Wait(); err != nil { return nil, fmt.Errorf("while waiting: %s", err) } return revisions, nil } func gitRevision(dirname, page, version string) string { cmd := exec.Command("git", "show", version+":"+page) cmd.Dir = dirname buf := bytes.Buffer{} cmd.Stdout = &buf cmd.Start() cmd.Wait() return buf.String() } func (fp *FilePages) RecentChanges() ([]Change, error) { cmd := exec.Command("git", "log", "--format=--1--%nDate: %aI%n--2--%n%b%n--3--", "--name-only") cmd.Dir = fp.dirname buf := bytes.Buffer{} cmd.Stdout = &buf err := cmd.Start() if err != nil { return nil, err } err = cmd.Wait() if err != nil { return nil, err } scanner := bufio.NewScanner(&buf) state := 0 var changes []Change var change Change body := "" for scanner.Scan() { line := scanner.Text() if line == "--1--" { state = 1 body = "" continue } if line == "--2--" { state = 2 continue } if line == "--3--" { state = 3 continue } if state == 1 && strings.HasPrefix(line, "Date: ") { line = line[6:] changeTime, err := time.Parse(time.RFC3339, line) if err != nil { return changes, err } change.Date = changeTime continue } if state == 2 { if line == "" { continue } body = body + line continue } if state == 3 { if line == "" { continue } change.Page = line } change.Body = body changes = append(changes, change) } return changes, nil } func (fp *FilePages) AllPages() ([]Page, error) { files, err := ioutil.ReadDir(fp.dirname) if err != nil { return nil, err } var pages []Page for _, file := range files { if file.Name()[0] == '.' || file.Name()[0] == '_' { continue } if file.Name() == "backrefs.json" { continue } pages = append(pages, fp.Get(file.Name())) } return pages, nil }