Save feeds to redis and return the feeds from redis

cmd/server/fetch.go

@@ -1,3 +1,4 @@
+// fetch url in different ways
 /*
    Microsub server
    Copyright (C) 2018  Peter Stuifzand
@@ -18,6 +19,7 @@
 package main
 
 import (
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"log"
@@ -27,6 +29,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/garyburd/redigo/redis"
 	"github.com/pstuifzand/microsub-server/microsub"
 	"willnorris.com/go/microformats"
 )
@@ -42,6 +45,111 @@ func init() {
 	cache = make(map[string]cacheItem)
 }
 
+// Fetch3 fills stuff
+func (b *memoryBackend) Fetch3(channel, fetchURL string) error {
+	log.Printf("Fetching channel=%s fetchURL=%s\n", channel, fetchURL)
+	channelKey := fmt.Sprintf("channel:%s:posts", channel)
+
+	md, err := Fetch2(fetchURL)
+	if err != nil {
+		return err
+	}
+
+	results := simplifyMicroformatData(md)
+
+	found := -1
+	for {
+		for i, r := range results {
+			if r["type"] == "card" {
+				found = i
+				break
+			}
+		}
+		if found >= 0 {
+			card := results[found]
+			results = append(results[:found], results[found+1:]...)
+			for i := range results {
+				if results[i]["type"] == "entry" && results[i]["author"] == card["url"] {
+					results[i]["author"] = card
+				}
+			}
+			found = -1
+			continue
+		}
+		break
+	}
+
+	for i, r := range results {
+		if as, ok := r["author"].(string); ok {
+			if r["type"] == "entry" && strings.HasPrefix(as, "http") {
+				md, _ := Fetch2(as)
+				author := simplifyMicroformatData(md)
+				for _, a := range author {
+					if a["type"] == "card" {
+						results[i]["author"] = a
+						break
+					}
+				}
+			}
+		}
+	}
+
+	// Filter items with "published" date
+	for _, r := range results {
+		r["_is_read"] = b.wasRead(channel, r)
+		if r["_is_read"].(bool) {
+			continue
+		}
+		if uid, e := r["uid"]; e {
+			r["_id"] = hex.EncodeToString([]byte(uid.(string)))
+		} else if uid, e := r["url"]; e {
+			r["_id"] = hex.EncodeToString([]byte(uid.(string)))
+		} else {
+			r["_id"] = "" // generate random value
+		}
+
+		if _, e := r["published"]; e {
+			item := mapToItem(r)
+
+			// send to redis
+
+			data, err := json.Marshal(item)
+			if err != nil {
+				log.Printf("error while creating item for redis: %v\n", err)
+				continue
+			}
+
+			forRedis := redisItem{
+				Id:        item.Id,
+				Published: item.Published,
+				Read:      item.Read,
+				Data:      data,
+			}
+			itemKey := fmt.Sprintf("item:%s", item.Id)
+			_, err = redis.String(b.Redis.Do("HMSET", redis.Args{}.Add(itemKey).AddFlat(&forRedis)...))
+			if err != nil {
+				log.Printf("error while writing item for redis: %v\n", err)
+				continue
+			}
+
+			_, err = b.Redis.Do("SADD", channelKey, itemKey)
+			if err != nil {
+				log.Printf("error while adding item %s to channel %s for redis: %v\n", itemKey, channelKey, err)
+				continue
+			}
+		}
+	}
+	return nil
+}
+
+type redisItem struct {
+	Id        string
+	Published string
+	Read      bool
+	Data      []byte
+}
+
+// Fetch2 fetches stuff
 func Fetch2(fetchURL string) (*microformats.Data, error) {
 	if !strings.HasPrefix(fetchURL, "http") {
 		return nil, fmt.Errorf("error parsing %s as url", fetchURL)

cmd/server/memory.go

@@ -26,7 +26,6 @@ import (
 	"net/url"
 	"os"
 	"reflect"
-	"sort"
 	"strings"
 
 	"github.com/garyburd/redigo/redis"
@@ -50,10 +49,16 @@ func (b *memoryBackend) Debug() {
 
 func (b *memoryBackend) load() {
 	filename := "backend.json"
-	f, _ := os.Open(filename)
+	f, err := os.Open(filename)
+	if err != nil {
+		panic("cant open backend.json")
+	}
 	defer f.Close()
 	jw := json.NewDecoder(f)
-	jw.Decode(b)
+	err = jw.Decode(b)
+	if err != nil {
+		panic("cant open backend.json")
+	}
 }
 
 func (b *memoryBackend) save() {
@@ -179,28 +184,41 @@ func mapToItem(result map[string]interface{}) microsub.Item {
 		}
 	}
 
-	if likeOf, e := result["like-of"]; e {
+	if value, e := result["like-of"]; e {
-		item.LikeOf = likeOf.([]string)
+		for _, v := range value.([]interface{}) {
+			item.LikeOf = append(item.LikeOf, v.(string))
+		}
 	}
 
 	if value, e := result["repost-of"]; e {
-		item.RepostOf = value.([]string)
+		for _, v := range value.([]interface{}) {
+			item.RepostOf = append(item.RepostOf, v.(string))
+		}
 	}
 
 	if value, e := result["bookmark-of"]; e {
-		item.BookmarkOf = value.([]string)
+		for _, v := range value.([]interface{}) {
+			item.BookmarkOf = append(item.BookmarkOf, v.(string))
+		}
 	}
 
 	if value, e := result["in-reply-to"]; e {
-		item.InReplyTo = value.([]string)
+		for _, v := range value.([]interface{}) {
+			item.InReplyTo = append(item.InReplyTo, v.(string))
+		}
 	}
 
 	if value, e := result["photo"]; e {
-		item.Photo = value.([]string)
+		for _, v := range value.([]interface{}) {
+			item.Photo = append(item.Photo, v.(string))
+		}
 	}
 
 	if value, e := result["category"]; e {
 		item.Category = value.([]string)
+		for _, v := range value.([]interface{}) {
+			item.Category = append(item.Category, v.(string))
+		}
 	}
 
 	if published, e := result["published"]; e {
@@ -222,80 +240,39 @@ func mapToItem(result map[string]interface{}) microsub.Item {
 }
 
 func (b *memoryBackend) TimelineGet(after, before, channel string) microsub.Timeline {
+	log.Printf("TimelineGet %s\n", channel)
 	feeds := b.FollowGetList(channel)
+	log.Println(feeds)
 
 	items := []microsub.Item{}
 
 	for _, feed := range feeds {
-		md, err := Fetch2(feed.URL)
+		b.Fetch3(channel, feed.URL)
-		if err == nil {
+	}
-			results := simplifyMicroformatData(md)
 
-			found := -1
+	channelKey := fmt.Sprintf("channel:%s:posts", channel)
-			for {
-				for i, r := range results {
-					if r["type"] == "card" {
-						found = i
-						break
-					}
-				}
-				if found >= 0 {
-					card := results[found]
-					results = append(results[:found], results[found+1:]...)
-					for i := range results {
-						if results[i]["type"] == "entry" && results[i]["author"] == card["url"] {
-							results[i]["author"] = card
-						}
-					}
-					found = -1
-					continue
-				}
-				break
-			}
 
-			for i, r := range results {
+	itemJsons, err := redis.ByteSlices(b.Redis.Do("SORT", channelKey, "BY", "*->Published", "GET", "*->Data", "DESC", "ALPHA"))
-				if as, ok := r["author"].(string); ok {
+	if err != nil {
-					if r["type"] == "entry" && strings.HasPrefix(as, "http") {
+		log.Println(err)
-						md, _ := Fetch2(as)
+		return microsub.Timeline{
-						author := simplifyMicroformatData(md)
+			Paging: microsub.Pagination{},
-						for _, a := range author {
+			Items:  items,
-							if a["type"] == "card" {
-								results[i]["author"] = a
-								break
-							}
-						}
-					}
-				}
-			}
-
-			// Filter items with "published" date
-			for _, r := range results {
-				r["_is_read"] = b.wasRead(channel, r)
-				if r["_is_read"].(bool) {
-					continue
-				}
-				if uid, e := r["uid"]; e {
-					r["_id"] = hex.EncodeToString([]byte(uid.(string)))
-				}
-				if uid, e := r["url"]; e {
-					r["_id"] = hex.EncodeToString([]byte(uid.(string)))
-				}
-
-				if _, e := r["published"]; e {
-					item := mapToItem(r)
-					items = append(items, item)
-				}
-			}
 		}
 	}
 
-	sort.SliceStable(items, func(a, b int) bool {
+	for _, obj := range itemJsons {
-		timeA := items[a].Published
+		item := microsub.Item{}
-		timeB := items[b].Published
+		json.Unmarshal(obj, &item)
-		return strings.Compare(timeA, timeB) > 0
+		items = append(items, item)
-	})
+	}
 
-	reverseSlice(items)
+	// sort.SliceStable(items, func(a, b int) bool {
+	// 	timeA := items[a].Published
+	// 	timeB := items[b].Published
+	// 	return strings.Compare(timeA, timeB) > 0
+	// })
+	// reverseSlice(items)
 
 	return microsub.Timeline{
 		Paging: microsub.Pagination{},