commit 6fe2eae1b6483d410ac7477e93ede0560eab76dc
Author: Peter Stuifzand <peter@stuifzand.eu>
Date:   Fri Feb 16 22:13:01 2018 +0100

    Add latest code and .drone.yml

diff --git a/.drone.yml b/.drone.yml
new file mode 100644
index 0000000..76d65ee
--- /dev/null
+++ b/.drone.yml
@@ -0,0 +1,22 @@
+pipeline:
+    build:
+        image: golang
+        commands:
+            - go get github.com/pstuifzand/microsub-server/cmd/server
+            - go build github.com/pstuifzand/microsub-server/cmd/server
+
+    publish:
+        image: plugins/docker
+        repo: registry.stuifzandapp.com/microsub-server
+        registry: registry.stuifzandapp.com
+        secrets: [ docker_username, docker_password ]
+
+    # deploy:
+    #     image: appleboy/drone-ssh
+    #     host: microsub.stuifzandapp.com
+    #     username: hub
+    #     secrets: ['ssh_key']
+    #     script:
+    #         - cd /home/hub/hub
+    #         - docker-compose pull
+    #         - docker-compose up -d
diff --git a/cmd/server/fetch.go b/cmd/server/fetch.go
new file mode 100644
index 0000000..5decff6
--- /dev/null
+++ b/cmd/server/fetch.go
@@ -0,0 +1,204 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+
+	"github.com/pstuifzand/microsub-server/microsub"
+	"willnorris.com/go/microformats"
+)
+
+var cache map[string]*microformats.Data
+
+func init() {
+	cache = make(map[string]*microformats.Data)
+}
+
+func Fetch2(fetchURL string) (*microformats.Data, error) {
+	if !strings.HasPrefix(fetchURL, "http") {
+		return nil, fmt.Errorf("error parsing %s as url", fetchURL)
+	}
+
+	u, err := url.Parse(fetchURL)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing %s as url: %s", fetchURL, err)
+	}
+
+	if data, e := cache[u.String()]; e {
+		log.Printf("HIT %s\n", u.String())
+		return data, nil
+	}
+
+	log.Printf("MISS %s\n", u.String())
+
+	resp, err := http.Get(u.String())
+	if err != nil {
+		return nil, fmt.Errorf("error while fetching %s: %s", u, err)
+	}
+
+	if !strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") {
+		return nil, fmt.Errorf("Content Type of %s = %s", fetchURL, resp.Header.Get("Content-Type"))
+	}
+
+	defer resp.Body.Close()
+	data := microformats.Parse(resp.Body, u)
+	cache[u.String()] = data
+	return data, nil
+}
+
+func Fetch(fetchURL string) []microsub.Item {
+	result := []microsub.Item{}
+
+	if !strings.HasPrefix(fetchURL, "http") {
+		return result
+	}
+
+	u, err := url.Parse(fetchURL)
+	if err != nil {
+		log.Printf("error parsing %s as url: %s", fetchURL, err)
+		return result
+	}
+	resp, err := http.Get(u.String())
+	if err != nil {
+		log.Printf("error while fetching %s: %s", u, err)
+		return result
+	}
+
+	if !strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") {
+		log.Printf("Content Type of %s = %s", fetchURL, resp.Header.Get("Content-Type"))
+		return result
+	}
+
+	defer resp.Body.Close()
+	data := microformats.Parse(resp.Body, u)
+	jw := json.NewEncoder(os.Stdout)
+	jw.SetIndent("", "    ")
+	jw.Encode(data)
+
+	author := microsub.Author{}
+
+	for _, item := range data.Items {
+		if item.Type[0] == "h-feed" {
+			for _, child := range item.Children {
+				previewItem := convertMfToItem(child)
+				result = append(result, previewItem)
+			}
+		} else if item.Type[0] == "h-card" {
+			mf := item
+			author.Filled = true
+			author.Type = "card"
+			for prop, value := range mf.Properties {
+				switch prop {
+				case "url":
+					author.URL = value[0].(string)
+					break
+				case "name":
+					author.Name = value[0].(string)
+					break
+				case "photo":
+					author.Photo = value[0].(string)
+					break
+				default:
+					fmt.Printf("prop name not implemented for author: %s with value %#v\n", prop, value)
+					break
+				}
+			}
+		} else if item.Type[0] == "h-entry" {
+			previewItem := convertMfToItem(item)
+			result = append(result, previewItem)
+		}
+	}
+
+	for i, item := range result {
+		if !item.Author.Filled {
+			result[i].Author = author
+		}
+	}
+
+	return result
+}
+
+func convertMfToItem(mf *microformats.Microformat) microsub.Item {
+	item := microsub.Item{}
+
+	item.Type = mf.Type[0]
+
+	for prop, value := range mf.Properties {
+		switch prop {
+		case "published":
+			item.Published = value[0].(string)
+			break
+		case "url":
+			item.URL = value[0].(string)
+			break
+		case "name":
+			item.Name = value[0].(string)
+			break
+		case "latitude":
+			item.Latitude = value[0].(string)
+			break
+		case "longitude":
+			item.Longitude = value[0].(string)
+			break
+		case "like-of":
+			for _, v := range value {
+				item.LikeOf = append(item.LikeOf, v.(string))
+			}
+			break
+		case "bookmark-of":
+			for _, v := range value {
+				item.BookmarkOf = append(item.BookmarkOf, v.(string))
+			}
+			break
+		case "in-reply-to":
+			for _, v := range value {
+				item.InReplyTo = append(item.InReplyTo, v.(string))
+			}
+			break
+		case "summary":
+			if content, ok := value[0].(map[string]interface{}); ok {
+				item.Content.HTML = content["html"].(string)
+				item.Content.Text = content["value"].(string)
+			} else if content, ok := value[0].(string); ok {
+				item.Content.Text = content
+			}
+			break
+		case "photo":
+			for _, v := range value {
+				item.Photo = append(item.Photo, v.(string))
+			}
+			break
+		case "category":
+			for _, v := range value {
+				item.Category = append(item.Category, v.(string))
+			}
+			break
+		case "content":
+			if content, ok := value[0].(map[string]interface{}); ok {
+				item.Content.HTML = content["html"].(string)
+				item.Content.Text = content["value"].(string)
+			} else if content, ok := value[0].(string); ok {
+				item.Content.Text = content
+			}
+			break
+		default:
+			fmt.Printf("prop name not implemented: %s with value %#v\n", prop, value)
+			break
+		}
+	}
+
+	if item.Name == strings.TrimSpace(item.Content.Text) {
+		item.Name = ""
+	}
+	if item.Content.HTML == "" && len(item.LikeOf) > 0 {
+		item.Name = ""
+	}
+
+	fmt.Printf("%#v\n", item)
+	return item
+}
diff --git a/cmd/server/main.go b/cmd/server/main.go
new file mode 100644
index 0000000..d8bb9b4
--- /dev/null
+++ b/cmd/server/main.go
@@ -0,0 +1,204 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+
+	"github.com/pstuifzand/microsub-server/microsub"
+	"willnorris.com/go/microformats"
+)
+
+type microsubHandler struct {
+	Backend microsub.Microsub
+}
+
+func simplify(item map[string][]interface{}) map[string]interface{} {
+	feedItem := make(map[string]interface{})
+
+	for k, v := range item {
+		if k == "bookmark-of" || k == "like-of" || k == "repost-of" || k == "in-reply-to" {
+			if value, ok := v[0].(*microformats.Microformat); ok {
+				m := simplify(value.Properties)
+				m["type"] = value.Type[0][2:]
+				feedItem[k] = []interface{}{m}
+			} else {
+				feedItem[k] = v
+			}
+		} else if k == "content" {
+			if content, ok := v[0].(map[string]interface{}); ok {
+				if text, e := content["value"]; e {
+					delete(content, "value")
+					if _, e := content["html"]; !e {
+						content["text"] = text
+					}
+				}
+				feedItem[k] = content
+			}
+		} else if k == "photo" {
+			feedItem[k] = v
+		} else if k == "video" {
+			feedItem[k] = v
+		} else if k == "featured" {
+			feedItem[k] = v
+		} else if value, ok := v[0].(*microformats.Microformat); ok {
+			m := simplify(value.Properties)
+			m["type"] = value.Type[0][2:]
+			feedItem[k] = m
+		} else if value, ok := v[0].(string); ok {
+			feedItem[k] = value
+		} else if value, ok := v[0].(map[string]interface{}); ok {
+			feedItem[k] = value
+		} else if value, ok := v[0].([]interface{}); ok {
+			feedItem[k] = value
+		}
+	}
+	return feedItem
+}
+
+func simplifyMicroformat(item *microformats.Microformat) map[string]interface{} {
+	newItem := simplify(item.Properties)
+	newItem["type"] = item.Type[0][2:]
+
+	children := []map[string]interface{}{}
+
+	if len(item.Children) > 0 {
+		for _, c := range item.Children {
+			child := simplifyMicroformat(c)
+			if c, e := child["children"]; e {
+				if ar, ok := c.([]map[string]interface{}); ok {
+					children = append(children, ar...)
+				}
+				delete(child, "children")
+			}
+			children = append(children, child)
+		}
+
+		newItem["children"] = children
+	}
+
+	return newItem
+}
+
+func simplifyMicroformatData(md *microformats.Data) []map[string]interface{} {
+	items := []map[string]interface{}{}
+	for _, item := range md.Items {
+		newItem := simplifyMicroformat(item)
+		items = append(items, newItem)
+		if c, e := newItem["children"]; e {
+			if ar, ok := c.([]map[string]interface{}); ok {
+				items = append(items, ar...)
+			}
+			delete(newItem, "children")
+		}
+	}
+	return items
+}
+
+func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	fmt.Println(r.URL.String())
+
+	if r.Method == http.MethodGet {
+		values := r.URL.Query()
+		action := values.Get("action")
+		if action == "channels" {
+			channels := h.Backend.ChannelsGetList()
+			jw := json.NewEncoder(w)
+			w.Header().Add("Content-Type", "application/json")
+			jw.Encode(map[string][]microsub.Channel{
+				"channels": channels,
+			})
+		} else if action == "timeline" {
+			timeline := h.Backend.TimelineGet(values.Get("after"), values.Get("before"), values.Get("channel"))
+			jw := json.NewEncoder(w)
+			w.Header().Add("Content-Type", "application/json")
+			jw.SetIndent("", "    ")
+			jw.Encode(timeline)
+		} else if action == "preview" {
+			md, err := Fetch2(values.Get("url"))
+			if err != nil {
+				http.Error(w, "Failed parsing url", 500)
+				return
+			}
+
+			results := simplifyMicroformatData(md)
+
+			jw := json.NewEncoder(w)
+			jw.SetIndent("", "    ")
+			w.Header().Add("Content-Type", "application/json")
+			jw.Encode(map[string]interface{}{
+				"items":  results,
+				"paging": microsub.Pagination{},
+			})
+		} else if action == "follow" {
+			channel := values.Get("channel")
+			following := h.Backend.FollowGetList(channel)
+			jw := json.NewEncoder(w)
+			w.Header().Add("Content-Type", "application/json")
+			jw.Encode(map[string][]microsub.Feed{
+				"items": following,
+			})
+		}
+		return
+	} else if r.Method == http.MethodPost {
+		values := r.URL.Query()
+		action := values.Get("action")
+		if action == "channels" {
+			name := values.Get("name")
+			method := values.Get("method")
+			uid := values.Get("channel")
+			if method == "delete" {
+				h.Backend.ChannelsDelete(uid)
+				w.Header().Add("Content-Type", "application/json")
+				fmt.Fprintln(w, "[]")
+				h.Backend.(Debug).Debug()
+				return
+			}
+
+			jw := json.NewEncoder(w)
+			if uid == "" {
+				channel := h.Backend.ChannelsCreate(name)
+				w.Header().Add("Content-Type", "application/json")
+				jw.Encode(channel)
+			} else {
+				channel := h.Backend.ChannelsUpdate(uid, name)
+				w.Header().Add("Content-Type", "application/json")
+				jw.Encode(channel)
+			}
+			h.Backend.(Debug).Debug()
+		} else if action == "follow" {
+			uid := values.Get("channel")
+			url := values.Get("url")
+
+			feed := h.Backend.FollowURL(uid, url)
+			w.Header().Add("Content-Type", "application/json")
+			jw := json.NewEncoder(w)
+			jw.Encode(feed)
+		} else if action == "unfollow" {
+			uid := values.Get("channel")
+			url := values.Get("url")
+			h.Backend.UnfollowURL(uid, url)
+			w.Header().Add("Content-Type", "application/json")
+			fmt.Fprintln(w, "[]")
+		} else if action == "search" {
+			query := values.Get("query")
+			feeds := h.Backend.Search(query)
+			jw := json.NewEncoder(w)
+			w.Header().Add("Content-Type", "application/json")
+			jw.Encode(map[string][]microsub.Feed{
+				"results": feeds,
+			})
+		}
+		return
+	}
+	return
+}
+
+func main() {
+	backend := loadMemoryBackend()
+	//backend := createMemoryBackend()
+
+	http.Handle("/microsub", &microsubHandler{backend})
+	log.Fatal(http.ListenAndServe(":80", nil))
+}
diff --git a/cmd/server/memory.go b/cmd/server/memory.go
new file mode 100644
index 0000000..0ef95e3
--- /dev/null
+++ b/cmd/server/memory.go
@@ -0,0 +1,209 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/pstuifzand/microsub-server/microsub"
+)
+
+type memoryBackend struct {
+	Channels map[string]microsub.Channel
+	Feeds    map[string][]microsub.Feed
+	//Items    map[string]map[string][]microsub.Item
+	NextUid int
+}
+
+type Debug interface {
+	Debug()
+}
+
+func (b *memoryBackend) Debug() {
+	fmt.Println(b.Channels)
+}
+
+func (b *memoryBackend) load() {
+	filename := "/tmp/backend.json"
+	f, _ := os.Open(filename)
+	defer f.Close()
+	jw := json.NewDecoder(f)
+	jw.Decode(b)
+}
+
+func (b *memoryBackend) save() {
+	filename := "/tmp/backend.json"
+	f, _ := os.Create(filename)
+	defer f.Close()
+	jw := json.NewEncoder(f)
+	jw.Encode(b)
+}
+
+func loadMemoryBackend() microsub.Microsub {
+	backend := &memoryBackend{}
+	backend.load()
+	return backend
+}
+
+func createMemoryBackend() microsub.Microsub {
+	backend := memoryBackend{}
+	defer backend.save()
+	backend.Channels = make(map[string]microsub.Channel)
+	backend.Feeds = make(map[string][]microsub.Feed)
+	channels := []microsub.Channel{
+		microsub.Channel{"0000", "default"},
+		microsub.Channel{"0001", "notifications"},
+		microsub.Channel{"1000", "Friends"},
+		microsub.Channel{"1001", "Family"},
+	}
+	for _, c := range channels {
+		backend.Channels[c.UID] = c
+	}
+	backend.NextUid = 1002
+	return &backend
+}
+
+// ChannelsGetList gets no channels
+func (b *memoryBackend) ChannelsGetList() []microsub.Channel {
+	channels := []microsub.Channel{}
+	for _, v := range b.Channels {
+		channels = append(channels, v)
+	}
+	return channels
+}
+
+// ChannelsCreate creates no channels
+func (b *memoryBackend) ChannelsCreate(name string) microsub.Channel {
+	defer b.save()
+	uid := fmt.Sprintf("%04d", b.NextUid)
+	channel := microsub.Channel{
+		UID:  uid,
+		Name: name,
+	}
+
+	b.Channels[channel.UID] = channel
+	b.Feeds[channel.UID] = []microsub.Feed{}
+	b.NextUid++
+	return channel
+}
+
+// ChannelsUpdate updates no channels
+func (b *memoryBackend) ChannelsUpdate(uid, name string) microsub.Channel {
+	defer b.save()
+	if c, e := b.Channels[uid]; e {
+		c.Name = name
+		b.Channels[uid] = c
+		return c
+	}
+	return microsub.Channel{}
+}
+
+func (b *memoryBackend) ChannelsDelete(uid string) {
+	defer b.save()
+	if _, e := b.Channels[uid]; e {
+		delete(b.Channels, uid)
+	}
+}
+
+func (b *memoryBackend) TimelineGet(after, before, channel string) microsub.Timeline {
+	feeds := b.FollowGetList(channel)
+
+	items := []map[string]interface{}{}
+
+	for _, feed := range feeds {
+		md, err := Fetch2(feed.URL)
+		if err == nil {
+			results := simplifyMicroformatData(md)
+
+			found := -1
+			for {
+				for i, r := range results {
+					if r["type"] == "card" {
+						found = i
+						break
+					}
+				}
+				if found >= 0 {
+					card := results[found]
+					results = append(results[:found], results[found+1:]...)
+					for i := range results {
+						if results[i]["type"] == "entry" && results[i]["author"] == card["url"] {
+							results[i]["author"] = card
+						}
+					}
+					found = -1
+					continue
+				}
+				break
+			}
+
+			for i, r := range results {
+				if as, ok := r["author"].(string); ok {
+					if r["type"] == "entry" && strings.HasPrefix(as, "http") {
+						md, _ := Fetch2(as)
+						author := simplifyMicroformatData(md)
+						for _, a := range author {
+							if a["type"] == "card" {
+								results[i]["author"] = a
+								break
+							}
+						}
+					}
+				}
+			}
+
+			items = append(items, results...)
+		}
+	}
+
+	return microsub.Timeline{
+		Paging: microsub.Pagination{},
+		Items:  items,
+	}
+}
+
+func (b *memoryBackend) FollowGetList(uid string) []microsub.Feed {
+	return b.Feeds[uid]
+}
+
+func (b *memoryBackend) FollowURL(uid string, url string) microsub.Feed {
+	defer b.save()
+	feed := microsub.Feed{"feed", url}
+	b.Feeds[uid] = append(b.Feeds[uid], feed)
+	return feed
+}
+
+func (b *memoryBackend) UnfollowURL(uid string, url string) {
+	defer b.save()
+	index := -1
+	for i, f := range b.Feeds[uid] {
+		if f.URL == url {
+			index = i
+			break
+		}
+	}
+	if index >= 0 {
+		feeds := b.Feeds[uid]
+		b.Feeds[uid] = append(feeds[:index], feeds[index+1:]...)
+	}
+}
+
+// TODO: improve search for feeds
+func (b *memoryBackend) Search(query string) []microsub.Feed {
+	return []microsub.Feed{
+		microsub.Feed{"feed", query},
+		//microsub.Feed{"feed", "https://peterstuifzand.nl/rss.xml"},
+	}
+}
+
+func (b *memoryBackend) PreviewURL(previewUrl string) microsub.Timeline {
+	md, err := Fetch2(previewUrl)
+	if err != nil {
+		return microsub.Timeline{}
+	}
+	results := simplifyMicroformatData(md)
+	return microsub.Timeline{
+		Items: results,
+	}
+}
diff --git a/cmd/server/null.go b/cmd/server/null.go
new file mode 100644
index 0000000..1199d73
--- /dev/null
+++ b/cmd/server/null.go
@@ -0,0 +1,69 @@
+package main
+
+import (
+	"github.com/pstuifzand/microsub-server/microsub"
+)
+
+// NullBackend is the simplest possible backend
+type NullBackend struct {
+}
+
+// ChannelsGetList gets no channels
+func (b *NullBackend) ChannelsGetList() []microsub.Channel {
+	return []microsub.Channel{
+		microsub.Channel{"0000", "default"},
+		microsub.Channel{"0001", "notifications"},
+		microsub.Channel{"1000", "Friends"},
+		microsub.Channel{"1001", "Family"},
+	}
+}
+
+// ChannelsCreate creates no channels
+func (b *NullBackend) ChannelsCreate(name string) microsub.Channel {
+	return microsub.Channel{
+		UID:  "1234",
+		Name: name,
+	}
+}
+
+// ChannelsUpdate updates no channels
+func (b *NullBackend) ChannelsUpdate(uid, name string) microsub.Channel {
+	return microsub.Channel{
+		UID:  uid,
+		Name: name,
+	}
+}
+
+// ChannelsDelete delets no channels
+func (b *NullBackend) ChannelsDelete(uid string) {
+}
+
+// TimelineGet gets no timeline
+func (b *NullBackend) TimelineGet(after, before, channel string) microsub.Timeline {
+	return microsub.Timeline{
+		Paging: microsub.Pagination{},
+		Items:  []map[string]interface{}{},
+	}
+}
+
+func (b *NullBackend) FollowGetList(uid string) []microsub.Feed {
+	return []microsub.Feed{}
+}
+
+func (b *NullBackend) FollowURL(uid string, url string) microsub.Feed {
+	return microsub.Feed{"feed", url}
+}
+
+func (b *NullBackend) UnfollowURL(uid string, url string) {
+}
+
+func (b *NullBackend) Search(query string) []microsub.Feed {
+	return []microsub.Feed{}
+}
+
+func (b *NullBackend) PreviewURL(url string) microsub.Timeline {
+	return microsub.Timeline{
+		Paging: microsub.Pagination{},
+		Items:  []map[string]interface{}{},
+	}
+}
diff --git a/cmd/server/server b/cmd/server/server
new file mode 100755
index 0000000..826da9e
Binary files /dev/null and b/cmd/server/server differ
diff --git a/microsub/protocol.go b/microsub/protocol.go
new file mode 100644
index 0000000..3c1c4de
--- /dev/null
+++ b/microsub/protocol.go
@@ -0,0 +1,84 @@
+package microsub
+
+/*
+	channels
+	search
+	preview
+	follow / unfollow
+	timeline
+	mute / unmute
+	block / unblock
+*/
+
+// Channel contains information about a channel.
+type Channel struct {
+	// UID is a unique id for the channel
+	UID  string `json:"uid"`
+	Name string `json:"name"`
+}
+
+type Author struct {
+	Filled bool   `json:"-"`
+	Type   string `json:"type"`
+	Name   string `json:"name"`
+	URL    string `json:"url"`
+	Photo  string `json:"photo"`
+}
+
+type Content struct {
+	Text string `json:"text"`
+	HTML string `json:"html"`
+}
+
+// Item is a post object
+type Item struct {
+	Type       string   `json:"type"`
+	Name       string   `json:"name,omitempty"`
+	Published  string   `json:"published"`
+	URL        string   `json:"url"`
+	Author     Author   `json:"author"`
+	Category   []string `json:"category"`
+	Photo      []string `json:"photo"`
+	LikeOf     []string `json:"like-of"`
+	BookmarkOf []string `json:"bookmark-of"`
+	InReplyTo  []string `json:"in-reply-to"`
+	Summary    []string `json:"summary,omitempty"`
+	Content    Content  `json:"content,omitempty"`
+	Latitude   string   `json:"latitude,omitempty"`
+	Longitude  string   `json:"longitude,omitempty"`
+}
+
+// Pagination contains information about paging
+type Pagination struct {
+	After  string `json:"after,omitempty"`
+	Before string `json:"before,omitempty"`
+}
+
+// Timeline is a combination of items and paging information
+type Timeline struct {
+	Items  []map[string]interface{} `json:"items"`
+	Paging Pagination               `json:"paging"`
+}
+
+type Feed struct {
+	Type string `json:"type"`
+	URL  string `json:"url"`
+}
+
+// Microsub is the main protocol that should be implemented by a backend
+type Microsub interface {
+	ChannelsGetList() []Channel
+	ChannelsCreate(name string) Channel
+	ChannelsUpdate(uid, name string) Channel
+	ChannelsDelete(uid string)
+
+	TimelineGet(before, after, channel string) Timeline
+
+	FollowGetList(uid string) []Feed
+	FollowURL(uid string, url string) Feed
+
+	UnfollowURL(uid string, url string)
+
+	Search(query string) []Feed
+	PreviewURL(url string) Timeline
+}