weekly/main.go

package main

import (
	"bytes"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/jinzhu/now"
	"p83.nl/go/ekster/pkg/jf2"
	"p83.nl/go/ekster/pkg/microsub"
	"willnorris.com/go/microformats"
)

type Summary struct {
	PostTypes map[string]string
	Name      string
	Published string
	Items     []microsub.Item
	Grouped   map[string][]microsub.Item
}

func init() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

func main() {
	u, err := url.Parse(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}
	filename := os.Args[2]

	re, err := regexp.Compile(`<span class="next-entry"></span>`)
	if err != nil {
		log.Fatal(err)
	}

	f, err := forceCreate(filename)

	data, err := ioutil.ReadAll(f)
	if err != nil {
		log.Fatal(err)
	}

	f.Close()

	var buf bytes.Buffer
	items, err := getEntriesForFeed(u)
	if err != nil {
		log.Fatal(err)
	}

	monday := now.With(time.Now()).Monday().AddDate(0, 0, -10)
	items = filterEntriesAfter(items, monday)

	summary := getSummary(monday, items)

	err = outputEntries(summary, &buf)
	if err != nil {
		log.Fatal(err)
	}

	output := re.ReplaceAll(data, buf.Bytes())
	err = ioutil.WriteFile(filename, output, 0666)
	if err != nil {
		log.Fatal(err)
	}
}

func getSummary(monday time.Time, items []microsub.Item) Summary {
	year, week := monday.ISOWeek()
	grouped := make(map[string][]microsub.Item)
	for _, item := range items {
		itemType := postTypeDiscovery(&item)
		grouped[itemType] = append(grouped[itemType], item)
	}
	summary := Summary{
		Name:      fmt.Sprintf("Digest for Week %d-%d", week, year),
		Published: monday.AddDate(0, 0, 7).Format("2 Jan 2006"),
		Items:     items,
		Grouped:   grouped,
	}
	return summary
}

func forceCreate(filename string) (io.ReadCloser, error) {
	t, err := template.ParseFiles("templates/full.html")
	if err != nil {
		return nil, err
	}
	f, err := os.Create(filename)
	if err != nil {
		return nil, err
	}
	err = t.Execute(f, nil)
	if err != nil {
		return nil, err
	}
	err = f.Close()
	if err != nil {
		return nil, err
	}
	f, err = os.Open(filename)
	if err != nil {
		return nil, err
	}
	return f, nil
}

func openAndCreate(filename string) (io.ReadCloser, error) {
	var f io.ReadCloser
	var err error

	f, err = os.Open(filename)

	if err != nil {
		if !os.IsExist(err) {
			f, err = forceCreate(filename)
		}
	}

	return f, err
}

func outputEntries(summary Summary, w io.Writer) error {
	summary.PostTypes = map[string]string{
		"checkin":     "Checkins",
		"event":       "Events",
		"repost-of":   "Reposts",
		"like-of":     "Likes",
		"in-reply-to": "Replies",
		"bookmark":    "Bookmarks",
		"photo":       "Photos",
		"note":        "Notes",
		"article":     "Articles",
	}

	t, err := template.ParseFiles("templates/weekly.html")
	if err != nil {
		log.Fatal(err)
	}
	err = t.Execute(w, summary)
	if err != nil {
		return err
	}
	return nil
}

func postTypeDiscovery(item *microsub.Item) string {
	if item == nil {
		return ""
	}
	if item.Checkin != nil {
		return "checkin"
	}
	if item.Type == "event" {
		return "event"
	}
	if len(item.RepostOf) > 0 && validUrl(item.RepostOf[0]) {
		return "repost-of"
	}
	if len(item.LikeOf) > 0 && validUrl(item.LikeOf[0]) {
		return "like-of"
	}
	if len(item.InReplyTo) > 0 && validUrl(item.InReplyTo[0]) {
		return "in-reply-to"
	}
	if len(item.BookmarkOf) > 0 && validUrl(item.BookmarkOf[0]) {
		return "bookmark"
	}
	if len(item.Photo) > 0 && validUrl(item.Photo[0]) {
		return "photo"
	}

	var content, name string
	if item.Content != nil {
		content = item.Content.Text
	}
	if content == "" {
		if item.Summary != "" {
			content = item.Summary
		}
	}

	if content == "" {
		return "note"
	}

	name = item.Name

	if name == "" {
		return "note"
	}

	name = strings.Join(strings.Fields(strings.TrimSpace(name)), " ")
	content = strings.Join(strings.Fields(strings.TrimSpace(content)), " ")

	if !strings.HasPrefix(content, name) {
		return "article"
	}

	return "note"
}

func validUrl(u string) bool {
	_, err := url.Parse(u)
	return err == nil
}

func getEntriesForFeed(u *url.URL) ([]microsub.Item, error) {
	resp, err := http.Get(u.String())
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("status is not 200, but %d", resp.StatusCode)
	}

	md := microformats.Parse(resp.Body, u)
	items := jf2.SimplifyMicroformatDataItems(md)
	return items, nil
}

func filterEntriesAfter(items []microsub.Item, from time.Time) []microsub.Item {
	var result []microsub.Item
	for _, item := range items {
		published, err := time.Parse(time.RFC3339, item.Published)
		if err != nil {
			continue
		}
		if published.After(from) {
			result = append(result, item)
		}
	}
	return result
}