From 6477288fee731860ffa3639d12d685b63e34f323 Mon Sep 17 00:00:00 2001 From: Peter Stuifzand Date: Sat, 7 Apr 2018 16:36:27 +0200 Subject: [PATCH] Save feeds to redis and return the feeds from redis --- cmd/server/fetch.go | 108 ++++++++++++++++++++++++++++++++++++++ cmd/server/memory.go | 121 ++++++++++++++++++------------------------- 2 files changed, 157 insertions(+), 72 deletions(-) diff --git a/cmd/server/fetch.go b/cmd/server/fetch.go index 08908af..4569673 100644 --- a/cmd/server/fetch.go +++ b/cmd/server/fetch.go @@ -1,3 +1,4 @@ +// fetch url in different ways /* Microsub server Copyright (C) 2018 Peter Stuifzand @@ -18,6 +19,7 @@ package main import ( + "encoding/hex" "encoding/json" "fmt" "log" @@ -27,6 +29,7 @@ import ( "strings" "time" + "github.com/garyburd/redigo/redis" "github.com/pstuifzand/microsub-server/microsub" "willnorris.com/go/microformats" ) @@ -42,6 +45,111 @@ func init() { cache = make(map[string]cacheItem) } +// Fetch3 fills stuff +func (b *memoryBackend) Fetch3(channel, fetchURL string) error { + log.Printf("Fetching channel=%s fetchURL=%s\n", channel, fetchURL) + channelKey := fmt.Sprintf("channel:%s:posts", channel) + + md, err := Fetch2(fetchURL) + if err != nil { + return err + } + + results := simplifyMicroformatData(md) + + found := -1 + for { + for i, r := range results { + if r["type"] == "card" { + found = i + break + } + } + if found >= 0 { + card := results[found] + results = append(results[:found], results[found+1:]...) + for i := range results { + if results[i]["type"] == "entry" && results[i]["author"] == card["url"] { + results[i]["author"] = card + } + } + found = -1 + continue + } + break + } + + for i, r := range results { + if as, ok := r["author"].(string); ok { + if r["type"] == "entry" && strings.HasPrefix(as, "http") { + md, _ := Fetch2(as) + author := simplifyMicroformatData(md) + for _, a := range author { + if a["type"] == "card" { + results[i]["author"] = a + break + } + } + } + } + } + + // Filter items with "published" date + for _, r := range results { + r["_is_read"] = b.wasRead(channel, r) + if r["_is_read"].(bool) { + continue + } + if uid, e := r["uid"]; e { + r["_id"] = hex.EncodeToString([]byte(uid.(string))) + } else if uid, e := r["url"]; e { + r["_id"] = hex.EncodeToString([]byte(uid.(string))) + } else { + r["_id"] = "" // generate random value + } + + if _, e := r["published"]; e { + item := mapToItem(r) + + // send to redis + + data, err := json.Marshal(item) + if err != nil { + log.Printf("error while creating item for redis: %v\n", err) + continue + } + + forRedis := redisItem{ + Id: item.Id, + Published: item.Published, + Read: item.Read, + Data: data, + } + itemKey := fmt.Sprintf("item:%s", item.Id) + _, err = redis.String(b.Redis.Do("HMSET", redis.Args{}.Add(itemKey).AddFlat(&forRedis)...)) + if err != nil { + log.Printf("error while writing item for redis: %v\n", err) + continue + } + + _, err = b.Redis.Do("SADD", channelKey, itemKey) + if err != nil { + log.Printf("error while adding item %s to channel %s for redis: %v\n", itemKey, channelKey, err) + continue + } + } + } + return nil +} + +type redisItem struct { + Id string + Published string + Read bool + Data []byte +} + +// Fetch2 fetches stuff func Fetch2(fetchURL string) (*microformats.Data, error) { if !strings.HasPrefix(fetchURL, "http") { return nil, fmt.Errorf("error parsing %s as url", fetchURL) diff --git a/cmd/server/memory.go b/cmd/server/memory.go index 5ded887..ce60899 100644 --- a/cmd/server/memory.go +++ b/cmd/server/memory.go @@ -26,7 +26,6 @@ import ( "net/url" "os" "reflect" - "sort" "strings" "github.com/garyburd/redigo/redis" @@ -50,10 +49,16 @@ func (b *memoryBackend) Debug() { func (b *memoryBackend) load() { filename := "backend.json" - f, _ := os.Open(filename) + f, err := os.Open(filename) + if err != nil { + panic("cant open backend.json") + } defer f.Close() jw := json.NewDecoder(f) - jw.Decode(b) + err = jw.Decode(b) + if err != nil { + panic("cant open backend.json") + } } func (b *memoryBackend) save() { @@ -179,28 +184,41 @@ func mapToItem(result map[string]interface{}) microsub.Item { } } - if likeOf, e := result["like-of"]; e { - item.LikeOf = likeOf.([]string) + if value, e := result["like-of"]; e { + for _, v := range value.([]interface{}) { + item.LikeOf = append(item.LikeOf, v.(string)) + } } if value, e := result["repost-of"]; e { - item.RepostOf = value.([]string) + for _, v := range value.([]interface{}) { + item.RepostOf = append(item.RepostOf, v.(string)) + } } if value, e := result["bookmark-of"]; e { - item.BookmarkOf = value.([]string) + for _, v := range value.([]interface{}) { + item.BookmarkOf = append(item.BookmarkOf, v.(string)) + } } if value, e := result["in-reply-to"]; e { - item.InReplyTo = value.([]string) + for _, v := range value.([]interface{}) { + item.InReplyTo = append(item.InReplyTo, v.(string)) + } } if value, e := result["photo"]; e { - item.Photo = value.([]string) + for _, v := range value.([]interface{}) { + item.Photo = append(item.Photo, v.(string)) + } } if value, e := result["category"]; e { item.Category = value.([]string) + for _, v := range value.([]interface{}) { + item.Category = append(item.Category, v.(string)) + } } if published, e := result["published"]; e { @@ -222,80 +240,39 @@ func mapToItem(result map[string]interface{}) microsub.Item { } func (b *memoryBackend) TimelineGet(after, before, channel string) microsub.Timeline { + log.Printf("TimelineGet %s\n", channel) feeds := b.FollowGetList(channel) + log.Println(feeds) items := []microsub.Item{} for _, feed := range feeds { - md, err := Fetch2(feed.URL) - if err == nil { - results := simplifyMicroformatData(md) + b.Fetch3(channel, feed.URL) + } - found := -1 - for { - for i, r := range results { - if r["type"] == "card" { - found = i - break - } - } - if found >= 0 { - card := results[found] - results = append(results[:found], results[found+1:]...) - for i := range results { - if results[i]["type"] == "entry" && results[i]["author"] == card["url"] { - results[i]["author"] = card - } - } - found = -1 - continue - } - break - } + channelKey := fmt.Sprintf("channel:%s:posts", channel) - for i, r := range results { - if as, ok := r["author"].(string); ok { - if r["type"] == "entry" && strings.HasPrefix(as, "http") { - md, _ := Fetch2(as) - author := simplifyMicroformatData(md) - for _, a := range author { - if a["type"] == "card" { - results[i]["author"] = a - break - } - } - } - } - } - - // Filter items with "published" date - for _, r := range results { - r["_is_read"] = b.wasRead(channel, r) - if r["_is_read"].(bool) { - continue - } - if uid, e := r["uid"]; e { - r["_id"] = hex.EncodeToString([]byte(uid.(string))) - } - if uid, e := r["url"]; e { - r["_id"] = hex.EncodeToString([]byte(uid.(string))) - } - - if _, e := r["published"]; e { - item := mapToItem(r) - items = append(items, item) - } - } + itemJsons, err := redis.ByteSlices(b.Redis.Do("SORT", channelKey, "BY", "*->Published", "GET", "*->Data", "DESC", "ALPHA")) + if err != nil { + log.Println(err) + return microsub.Timeline{ + Paging: microsub.Pagination{}, + Items: items, } } - sort.SliceStable(items, func(a, b int) bool { - timeA := items[a].Published - timeB := items[b].Published - return strings.Compare(timeA, timeB) > 0 - }) + for _, obj := range itemJsons { + item := microsub.Item{} + json.Unmarshal(obj, &item) + items = append(items, item) + } - reverseSlice(items) + // sort.SliceStable(items, func(a, b int) bool { + // timeA := items[a].Published + // timeB := items[b].Published + // return strings.Compare(timeA, timeB) > 0 + // }) + // reverseSlice(items) return microsub.Timeline{ Paging: microsub.Pagination{},