peter/ekster
1
0
Fork 0
Code Issues 3 Pull Requests Projects 3 Releases 1 Wiki Activity

Run fetch every hour

Browse Source
This commit is contained in:
Peter Stuifzand 2018-04-07 20:00:48 +02:00
parent 7baf3a9ce5
commit 42cee3c9a4
2 changed files with 40 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/server/main.go
View File

@ -370,5 +370,6 @@ func main() {
http.Handle("/incoming/", &incomingHandler{ http.Handle("/incoming/", &incomingHandler{
Backend: &hubBackend, Backend: &hubBackend,
}) })
backend.(*memoryBackend).run()
log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), nil)) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), nil))
} }

48
cmd/server/memory.go
View File

@ -27,6 +27,7 @@ import (
"os" "os"
"reflect" "reflect"
"strings" "strings"
"time"
"github.com/garyburd/redigo/redis" "github.com/garyburd/redigo/redis"
"github.com/pstuifzand/microsub-server/microsub" "github.com/pstuifzand/microsub-server/microsub"
@ -37,6 +38,9 @@ type memoryBackend struct {
Channels map[string]microsub.Channel Channels map[string]microsub.Channel
Feeds map[string][]microsub.Feed Feeds map[string][]microsub.Feed
NextUid int NextUid int
ticker *time.Ticker
quit chan struct{}
} }
type Debug interface { type Debug interface {
@ -239,6 +243,27 @@ func mapToItem(result map[string]interface{}) microsub.Item {
return item return item
} }
func (b *memoryBackend) run() {
b.ticker = time.NewTicker(1 * time.Hour)
b.quit = make(chan struct{})
go func() {
for {
select {
case <-b.ticker.C:
for uid, _ := range b.Channels {
for _, feed := range b.Feeds[uid] {
b.Fetch3(uid, feed.URL)
}
}
case <-b.quit:
b.ticker.Stop()
return
}
}
}()
}
func (b *memoryBackend) TimelineGet(after, before, channel string) microsub.Timeline { func (b *memoryBackend) TimelineGet(after, before, channel string) microsub.Timeline {
log.Printf("TimelineGet %s\n", channel) log.Printf("TimelineGet %s\n", channel)
feeds := b.FollowGetList(channel) feeds := b.FollowGetList(channel)
@ -246,9 +271,9 @@ func (b *memoryBackend) TimelineGet(after, before, channel string) microsub.Time
items := []microsub.Item{} items := []microsub.Item{}
for _, feed := range feeds { // for _, feed := range feeds {
b.Fetch3(channel, feed.URL) // b.Fetch3(channel, feed.URL)
} // }
channelKey := fmt.Sprintf("channel:%s:posts", channel) channelKey := fmt.Sprintf("channel:%s:posts", channel)
@ -283,7 +308,7 @@ func reverseSlice(s interface{}) {
} }
func (b *memoryBackend) checkRead(channel string, uid string) bool { func (b *memoryBackend) checkRead(channel string, uid string) bool {
args := redis.Args{}.Add(fmt.Sprintf("timeline:%s:read", channel)).Add(uid) args := redis.Args{}.Add(fmt.Sprintf("timeline:%s:read", channel)).Add("item:" + uid)
member, err := redis.Bool(b.Redis.Do("SISMEMBER", args...)) member, err := redis.Bool(b.Redis.Do("SISMEMBER", args...))
if err != nil { if err != nil {
log.Printf("Checking read for channel %s item %s has failed\n", channel, uid) log.Printf("Checking read for channel %s item %s has failed\n", channel, uid)
@ -410,7 +435,6 @@ func (b *memoryBackend) PreviewURL(previewURL string) microsub.Timeline {
items := []microsub.Item{} items := []microsub.Item{}
for _, r := range results { for _, r := range results {
item := mapToItem(r) item := mapToItem(r)
log.Println(item)
items = append(items, item) items = append(items, item)
} }
return microsub.Timeline{ return microsub.Timeline{
@ -418,11 +442,17 @@ func (b *memoryBackend) PreviewURL(previewURL string) microsub.Timeline {
} }
} }
func (b *memoryBackend) MarkRead(channel string, itemUids []string) { func (b *memoryBackend) MarkRead(channel string, uids []string) {
log.Printf("Marking read for %s %v\n", channel, itemUids) log.Printf("Marking read for %s %v\n", channel, uids)
args := redis.Args{}.Add(fmt.Sprintf("timeline:%s:read", channel)).AddFlat(itemUids)
itemUIDs := []string{}
for _, uid := range uids {
itemUIDs = append(itemUIDs, "item:"+uid)
}
args := redis.Args{}.Add(fmt.Sprintf("timeline:%s:read", channel)).AddFlat(itemUIDs)
if _, err := b.Redis.Do("SADD", args...); err != nil { if _, err := b.Redis.Do("SADD", args...); err != nil {
log.Printf("Marking read for channel %s has failed\n", channel) log.Printf("Marking read for channel %s has failed\n", channel)
} }
log.Printf("Marking read success for %s %v\n", channel, itemUids) log.Printf("Marking read success for %s %v\n", channel, itemUIDs)
} }