Use items for instead of maps

cmd/server/auth.go

@@ -0,0 +1,106 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"regexp"
+	"time"
+
+	"github.com/garyburd/redigo/redis"
+)
+
+// TokenResponse is the information that we get back from the token endpoint of the user...
+type TokenResponse struct {
+	Me       string `json:"me"`
+	ClientID string `json:"client_id"`
+	Scope    string `json:"scope"`
+	IssuedAt int64  `json:"issued_at"`
+	Nonce    int64  `json:"nonce"`
+}
+
+var authHeaderRegex = regexp.MustCompile("^Bearer (.+)$")
+
+func (h *microsubHandler) cachedCheckAuthToken(header string, r *TokenResponse) bool {
+	r.Me = "https://publog.stuifzandapp.com/"
+	log.Println("Cached checking Auth Token")
+
+	tokens := authHeaderRegex.FindStringSubmatch(header)
+	if len(tokens) != 2 {
+		log.Println("No token found in the header")
+		return false
+	}
+	key := fmt.Sprintf("token:%s", tokens[1])
+
+	var err error
+
+	values, err := redis.Values(h.Redis.Do("HGETALL", key))
+	if err == nil && len(values) > 0 {
+		if err = redis.ScanStruct(values, r); err == nil {
+			return true
+		}
+	} else {
+		log.Printf("Error while HGETALL %v\n", err)
+	}
+
+	authorized := h.checkAuthToken(header, r)
+	authorized = true
+
+	if authorized {
+		fmt.Printf("Token response: %#v\n", r)
+		_, err = h.Redis.Do("HMSET", redis.Args{}.Add(key).AddFlat(r)...)
+		if err != nil {
+			log.Printf("Error while setting token: %v\n", err)
+			return authorized
+		}
+		_, err = h.Redis.Do("EXPIRE", key, uint64(10*time.Minute/time.Second))
+		if err != nil {
+			log.Printf("Error while setting expire on token: %v\n", err)
+			log.Println("Deleting token")
+			_, err = h.Redis.Do("DEL", key)
+			if err != nil {
+				log.Printf("Deleting token failed: %v", err)
+			}
+			return authorized
+		}
+	}
+
+	return authorized
+}
+
+func (h *microsubHandler) checkAuthToken(header string, token *TokenResponse) bool {
+	log.Println("Checking auth token")
+	req, err := http.NewRequest("GET", "https://publog.stuifzandapp.com/authtoken", nil)
+	if err != nil {
+		log.Println(err)
+		return false
+	}
+
+	req.Header.Add("Authorization", header)
+	req.Header.Add("Accept", "application/json")
+
+	client := http.Client{}
+	res, err := client.Do(req)
+	if err != nil {
+		log.Println(err)
+		return false
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode < 200 || res.StatusCode >= 300 {
+		log.Printf("HTTP StatusCode when verifying token: %d\n", res.StatusCode)
+		return false
+	}
+
+	dec := json.NewDecoder(res.Body)
+	err = dec.Decode(&token)
+
+	if err != nil {
+		log.Printf("Error in json object: %v", err)
+		return false
+	}
+
+	log.Println("Auth Token: Success")
+	return true
+}

cmd/server/fetch.go

@@ -64,7 +64,7 @@ func Fetch2(fetchURL string) (*microformats.Data, error) {
 		log.Printf("MISS %s\n", u.String())
 	}
 
-	resp, err := http.Get(u.String())
+	resp, err := http.Head(u.String())
 	if err != nil {
 		return nil, fmt.Errorf("error while fetching %s: %s", u, err)
 	}

cmd/server/incoming.go

@@ -14,8 +14,9 @@ import (
 
 // HubBackend handles information for the incoming handler
 type HubBackend interface {
-	CreateFeed(url string) int64
+	CreateFeed(url string, contentType string) (int64, error)
 	GetSecret(id int64) string
+	UpdateFeed(feedID int64, contentType string, content []byte) error
 }
 
 type incomingHandler struct {
@@ -93,12 +94,17 @@ func (h *incomingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	ct := r.Header.Get("Content-Type")
 	if strings.HasPrefix(ct, "application/rss+xml") {
 		// RSS parsing
+		h.Backend.UpdateFeed(feed, ct, feedContent)
 	} else if strings.HasPrefix(ct, "application/atom+xml") {
 		// Atom parsing
+		h.Backend.UpdateFeed(feed, ct, feedContent)
 	} else if strings.HasPrefix(ct, "text/html") {
 		// h-entry parsing
+		h.Backend.UpdateFeed(feed, ct, feedContent)
 	} else {
-		http.Error(w, "Unknown format of body", 400)
+		http.Error(w, fmt.Sprintf("Unknown format of body: %s", ct), 400)
 		return
 	}
+
+	return
 }

cmd/server/main.go

@@ -27,12 +27,12 @@ import (
 	"net/url"
 	"os"
 	"regexp"
-	"strings"
 	"time"
 
+	"rss"
+
 	"github.com/garyburd/redigo/redis"
 	"github.com/pstuifzand/microsub-server/microsub"
-	"willnorris.com/go/microformats"
 )
 
 var (
@@ -74,19 +74,21 @@ func (h *hubIncomingBackend) GetSecret(id int64) string {
 	return secret
 }
 
-var hubUrl = "https://hub.stuifzandapp.com/"
+var hubURL = "https://hub.stuifzandapp.com/"
 
-func (h *hubIncomingBackend) CreateFeed(topic string) int64 {
+func (h *hubIncomingBackend) CreateFeed(topic string, contentType string) (int64, error) {
 	id, err := redis.Int64(h.conn.Do("INCR", "feed:next_id"))
+
 	if err != nil {
-		log.Println(err)
+		return 0, err
 	}
 
 	h.conn.Do("HSET", fmt.Sprintf("feed:%d", id), "url", topic)
+	h.conn.Do("HSET", fmt.Sprintf("feed:%d", id), "type", contentType)
 	secret := randStringBytes(16)
 	h.conn.Do("HSET", fmt.Sprintf("feed:%d", id), "secret", secret)
 
-	hub, err := url.Parse(hubUrl)
+	hub, err := url.Parse(hubURL)
 	q := hub.Query()
 	q.Add("hub.mode", "subscribe")
 	q.Add("hub.callback", fmt.Sprintf("https://microsub.stuifzandapp.com/incoming/%d", id))
@@ -99,211 +101,87 @@ func (h *hubIncomingBackend) CreateFeed(topic string) int64 {
 	res, err := client.PostForm(hub.String(), q)
 	if err != nil {
 		log.Printf("new request: %s\n", err)
-		return -1
+		return 0, err
 	}
 	defer res.Body.Close()
 
-	fmt.Println(res)
+	filename := fmt.Sprintf("backend/feeds/%d.json", id)
 
-	return id
+	feed, err := rss.Fetch(topic)
-}
-
-func simplify(itemType string, item map[string][]interface{}) map[string]interface{} {
-	feedItem := make(map[string]interface{})
-
-	for k, v := range item {
-		if k == "bookmark-of" || k == "like-of" || k == "repost-of" || k == "in-reply-to" {
-			if value, ok := v[0].(*microformats.Microformat); ok {
-
-				mType := value.Type[0][2:]
-				m := simplify(mType, value.Properties)
-				m["type"] = mType
-				feedItem[k] = []interface{}{m}
-			} else {
-				feedItem[k] = v
-			}
-		} else if k == "content" {
-			if content, ok := v[0].(map[string]interface{}); ok {
-				if text, e := content["value"]; e {
-					delete(content, "value")
-					content["text"] = text
-				}
-				feedItem[k] = content
-			}
-		} else if k == "photo" {
-			if itemType == "card" {
-				if len(v) >= 1 {
-					if value, ok := v[0].(string); ok {
-						feedItem[k] = value
-					}
-				}
-			} else {
-				feedItem[k] = v
-			}
-		} else if k == "video" {
-			feedItem[k] = v
-		} else if k == "featured" {
-			feedItem[k] = v
-		} else if value, ok := v[0].(*microformats.Microformat); ok {
-			mType := value.Type[0][2:]
-			m := simplify(mType, value.Properties)
-			m["type"] = mType
-			feedItem[k] = m
-		} else if value, ok := v[0].(string); ok {
-			feedItem[k] = value
-		} else if value, ok := v[0].(map[string]interface{}); ok {
-			feedItem[k] = value
-		} else if value, ok := v[0].([]interface{}); ok {
-			feedItem[k] = value
-		}
-	}
-
-	// Remove "name" when it's equals to "content[text]"
-	if name, e := feedItem["name"]; e {
-		if content, e2 := feedItem["content"]; e2 {
-			if contentMap, ok := content.(map[string]interface{}); ok {
-				if text, e3 := contentMap["text"]; e3 {
-					if strings.TrimSpace(name.(string)) == strings.TrimSpace(text.(string)) {
-						delete(feedItem, "name")
-					}
-				}
-			}
-		}
-	}
-
-	return feedItem
-}
-
-func simplifyMicroformat(item *microformats.Microformat) map[string]interface{} {
-	itemType := item.Type[0][2:]
-	newItem := simplify(itemType, item.Properties)
-	newItem["type"] = itemType
-
-	children := []map[string]interface{}{}
-
-	if len(item.Children) > 0 {
-		for _, c := range item.Children {
-			child := simplifyMicroformat(c)
-			if c, e := child["children"]; e {
-				if ar, ok := c.([]map[string]interface{}); ok {
-					children = append(children, ar...)
-				}
-				delete(child, "children")
-			}
-			children = append(children, child)
-		}
-
-		newItem["children"] = children
-	}
-
-	return newItem
-}
-
-func simplifyMicroformatData(md *microformats.Data) []map[string]interface{} {
-	items := []map[string]interface{}{}
-	for _, item := range md.Items {
-		newItem := simplifyMicroformat(item)
-		items = append(items, newItem)
-		if c, e := newItem["children"]; e {
-			if ar, ok := c.([]map[string]interface{}); ok {
-				items = append(items, ar...)
-			}
-			delete(newItem, "children")
-		}
-	}
-	return items
-}
-
-// TokenResponse is the information that we get back from the token endpoint of the user...
-type TokenResponse struct {
-	Me       string `json:"me"`
-	ClientID string `json:"client_id"`
-	Scope    string `json:"scope"`
-	IssuedAt int64  `json:"issued_at"`
-	Nonce    int64  `json:"nonce"`
-}
-
-var authHeaderRegex = regexp.MustCompile("^Bearer (.+)$")
-
-func (h *microsubHandler) cachedCheckAuthToken(header string, r *TokenResponse) bool {
-	log.Println("Cached checking Auth Token")
-
-	tokens := authHeaderRegex.FindStringSubmatch(header)
-	if len(tokens) != 2 {
-		log.Println("No token found in the header")
-		return false
-	}
-	key := fmt.Sprintf("token:%s", tokens[1])
-
-	var err error
-
-	values, err := redis.Values(h.Redis.Do("HGETALL", key))
-	if err == nil && len(values) > 0 {
-		if err = redis.ScanStruct(values, r); err == nil {
-			return true
-		}
-	} else {
-		log.Printf("Error while HGETALL %v\n", err)
-	}
-
-	authorized := h.checkAuthToken(header, r)
-
-	if authorized {
-		fmt.Printf("Token response: %#v\n", r)
-		_, err = h.Redis.Do("HMSET", redis.Args{}.Add(key).AddFlat(r)...)
-		if err != nil {
-			log.Printf("Error while setting token: %v\n", err)
-			return authorized
-		}
-		_, err = h.Redis.Do("EXPIRE", key, uint64(10*time.Minute/time.Second))
-		if err != nil {
-			log.Printf("Error while setting expire on token: %v\n", err)
-			log.Println("Deleting token")
-			_, err = h.Redis.Do("DEL", key)
-			if err != nil {
-				log.Printf("Deleting token failed: %v", err)
-			}
-			return authorized
-		}
-	}
-
-	return authorized
-}
-
-func (h *microsubHandler) checkAuthToken(header string, token *TokenResponse) bool {
-	log.Println("Checking auth token")
-	req, err := http.NewRequest("GET", "https://publog.stuifzandapp.com/authtoken", nil)
 	if err != nil {
-		log.Println(err)
+		return 0, err
-		return false
 	}
+	os.MkdirAll("backend/feeds", 0755)
 
-	req.Header.Add("Authorization", header)
+	f, err := os.Create(filename)
-	req.Header.Add("Accept", "application/json")
-
-	client := http.Client{}
-	res, err := client.Do(req)
 	if err != nil {
-		log.Println(err)
+		return 0, err
-		return false
-	}
-	defer res.Body.Close()
-
-	if res.StatusCode < 200 || res.StatusCode >= 300 {
-		log.Printf("HTTP StatusCode when verifying token: %d\n", res.StatusCode)
-		return false
 	}
 
-	dec := json.NewDecoder(res.Body)
+	defer f.Close()
-	err = dec.Decode(&token)
 
+	out := json.NewEncoder(f)
+	err = out.Encode(feed)
 	if err != nil {
-		log.Printf("Error in json object: %v", err)
+		return 0, err
-		return false
 	}
 
-	log.Println("Auth Token: Success")
+	return id, nil
-	return true
+}
+
+func readFeedFile(filename string) (*rss.Feed, error) {
+	f, err := os.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	var feed *rss.Feed
+	out := json.NewDecoder(f)
+	err = out.Decode(feed)
+	if err != nil {
+		return nil, err
+	}
+	return feed, err
+}
+
+func writeFeedFile(filename string, feed *rss.Feed) error {
+	f, err := os.Create(filename)
+	if err != nil {
+		return err
+	}
+
+	defer f.Close()
+
+	out := json.NewEncoder(f)
+	err = out.Encode(feed)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (h *hubIncomingBackend) UpdateFeed(feedID int64, contentType string, content []byte) error {
+	_, err := redis.String(h.conn.Do("HGET", fmt.Sprintf("feed:%d", feedID), "url"))
+	if err != nil {
+		return err
+	}
+
+	filename := fmt.Sprintf("backend/feeds/%d.json", feedID)
+
+	feed, err := readFeedFile(filename)
+	if err != nil {
+		return err
+	}
+
+	err = feed.UpdateWithContent(content)
+	if err != nil {
+		return err
+	}
+
+	err = writeFeedFile(filename, feed)
+
+	return err
 }
 
 func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -350,6 +228,7 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		} else if action == "preview" {
 			md, err := Fetch2(values.Get("url"))
 			if err != nil {
+				log.Println(err)
 				http.Error(w, "Failed parsing url", 500)
 				return
 			}
@@ -404,7 +283,7 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		} else if action == "follow" {
 			uid := values.Get("channel")
 			url := values.Get("url")
-			h.HubIncomingBackend.CreateFeed(url)
+			h.HubIncomingBackend.CreateFeed(url, "application/rss+xml")
 			feed := h.Backend.FollowURL(uid, url)
 			w.Header().Add("Content-Type", "application/json")
 			jw := json.NewEncoder(w)
@@ -487,10 +366,11 @@ func main() {
 
 	if createBackend {
 		backend = createMemoryBackend()
-	} else {
+		return
-		backend = loadMemoryBackend(conn)
 	}
 
+	backend = loadMemoryBackend(conn)
+
 	hubBackend := hubIncomingBackend{conn}
 
 	http.Handle("/microsub", &microsubHandler{

cmd/server/memory.go

@@ -130,10 +130,35 @@ func (b *memoryBackend) ChannelsDelete(uid string) {
 	}
 }
 
+func mapToItem(result map[string]interface{}) microsub.Item {
+	item := microsub.Item{}
+
+	if name, e := result["name"]; e {
+		item.Name = name.(string)
+	}
+
+	if content, e := result["content"]; e {
+		if c, ok := content.(map[string]interface{}); ok {
+			if html, e2 := c["html"]; e2 {
+				item.Content.HTML = html.(string)
+			}
+			if text, e2 := c["value"]; e2 {
+				item.Content.Text = text.(string)
+			}
+		}
+	}
+
+	if published, e := result["published"]; e {
+		item.Published = published.(string)
+	}
+
+	return item
+}
+
 func (b *memoryBackend) TimelineGet(after, before, channel string) microsub.Timeline {
 	feeds := b.FollowGetList(channel)
 
-	items := []map[string]interface{}{}
+	items := []microsub.Item{}
 
 	for _, feed := range feeds {
 		md, err := Fetch2(feed.URL)
@@ -191,15 +216,16 @@ func (b *memoryBackend) TimelineGet(after, before, channel string) microsub.Time
 				}
 
 				if _, e := r["published"]; e {
-					items = append(items, r)
+					item := mapToItem(r)
+					items = append(items, item)
 				}
 			}
 		}
 	}
 
 	sort.SliceStable(items, func(a, b int) bool {
-		timeA, _ := items[a]["published"].(string)
+		timeA := items[a].Published
-		timeB, _ := items[b]["published"].(string)
+		timeB := items[b].Published
 		return strings.Compare(timeA, timeB) > 0
 	})
 
@@ -282,8 +308,13 @@ func (b *memoryBackend) PreviewURL(previewURL string) microsub.Timeline {
 		return microsub.Timeline{}
 	}
 	results := simplifyMicroformatData(md)
+	items := []microsub.Item{}
+	for _, r := range results {
+		item := mapToItem(r)
+		items = append(items, item)
+	}
 	return microsub.Timeline{
-		Items: results,
+		Items: items,
 	}
 }
 

cmd/server/null.go

@@ -59,7 +59,7 @@ func (b *NullBackend) ChannelsDelete(uid string) {
 func (b *NullBackend) TimelineGet(after, before, channel string) microsub.Timeline {
 	return microsub.Timeline{
 		Paging: microsub.Pagination{},
-		Items:  []map[string]interface{}{},
+		Items:  []microsub.Item{},
 	}
 }
 
@@ -81,7 +81,7 @@ func (b *NullBackend) Search(query string) []microsub.Feed {
 func (b *NullBackend) PreviewURL(url string) microsub.Timeline {
 	return microsub.Timeline{
 		Paging: microsub.Pagination{},
-		Items:  []map[string]interface{}{},
+		Items:  []microsub.Item{},
 	}
 }
 

cmd/server/simplify.go

@@ -0,0 +1,113 @@
+package main
+
+import (
+	"strings"
+
+	"willnorris.com/go/microformats"
+)
+
+func simplify(itemType string, item map[string][]interface{}) map[string]interface{} {
+	feedItem := make(map[string]interface{})
+
+	for k, v := range item {
+		if k == "bookmark-of" || k == "like-of" || k == "repost-of" || k == "in-reply-to" {
+			if value, ok := v[0].(*microformats.Microformat); ok {
+
+				mType := value.Type[0][2:]
+				m := simplify(mType, value.Properties)
+				m["type"] = mType
+				feedItem[k] = []interface{}{m}
+			} else {
+				feedItem[k] = v
+			}
+		} else if k == "content" {
+			if content, ok := v[0].(map[string]interface{}); ok {
+				if text, e := content["value"]; e {
+					delete(content, "value")
+					content["text"] = text
+				}
+				feedItem[k] = content
+			}
+		} else if k == "photo" {
+			if itemType == "card" {
+				if len(v) >= 1 {
+					if value, ok := v[0].(string); ok {
+						feedItem[k] = value
+					}
+				}
+			} else {
+				feedItem[k] = v
+			}
+		} else if k == "video" {
+			feedItem[k] = v
+		} else if k == "featured" {
+			feedItem[k] = v
+		} else if value, ok := v[0].(*microformats.Microformat); ok {
+			mType := value.Type[0][2:]
+			m := simplify(mType, value.Properties)
+			m["type"] = mType
+			feedItem[k] = m
+		} else if value, ok := v[0].(string); ok {
+			feedItem[k] = value
+		} else if value, ok := v[0].(map[string]interface{}); ok {
+			feedItem[k] = value
+		} else if value, ok := v[0].([]interface{}); ok {
+			feedItem[k] = value
+		}
+	}
+
+	// Remove "name" when it's equals to "content[text]"
+	if name, e := feedItem["name"]; e {
+		if content, e2 := feedItem["content"]; e2 {
+			if contentMap, ok := content.(map[string]interface{}); ok {
+				if text, e3 := contentMap["text"]; e3 {
+					if strings.TrimSpace(name.(string)) == strings.TrimSpace(text.(string)) {
+						delete(feedItem, "name")
+					}
+				}
+			}
+		}
+	}
+
+	return feedItem
+}
+
+func simplifyMicroformat(item *microformats.Microformat) map[string]interface{} {
+	itemType := item.Type[0][2:]
+	newItem := simplify(itemType, item.Properties)
+	newItem["type"] = itemType
+
+	children := []map[string]interface{}{}
+
+	if len(item.Children) > 0 {
+		for _, c := range item.Children {
+			child := simplifyMicroformat(c)
+			if c, e := child["children"]; e {
+				if ar, ok := c.([]map[string]interface{}); ok {
+					children = append(children, ar...)
+				}
+				delete(child, "children")
+			}
+			children = append(children, child)
+		}
+
+		newItem["children"] = children
+	}
+
+	return newItem
+}
+
+func simplifyMicroformatData(md *microformats.Data) []map[string]interface{} {
+	items := []map[string]interface{}{}
+	for _, item := range md.Items {
+		newItem := simplifyMicroformat(item)
+		items = append(items, newItem)
+		if c, e := newItem["children"]; e {
+			if ar, ok := c.([]map[string]interface{}); ok {
+				items = append(items, ar...)
+			}
+			delete(newItem, "children")
+		}
+	}
+	return items
+}

microsub/protocol.go

@@ -75,8 +75,8 @@ type Pagination struct {
 
 // Timeline is a combination of items and paging information
 type Timeline struct {
-	Items  []map[string]interface{} `json:"items"`
+	Items  []Item     `json:"items"`
-	Paging Pagination               `json:"paging"`
+	Paging Pagination `json:"paging"`
 }
 
 type Feed struct {