From 34133191fc9d66774a6973531be3ab667d038c85 Mon Sep 17 00:00:00 2001 From: Peter Stuifzand Date: Tue, 18 Dec 2018 19:40:56 +0100 Subject: [PATCH] Cleanup (parse category better) --- cmd/eksterd/hubbackend.go | 38 ++++++++++++++++++++------------------ pkg/fetch/fetch.go | 4 +--- pkg/jf2/simplify.go | 19 ++++++++++++------- pkg/jsonfeed/jsonfeed.go | 13 +++++++++++++ 4 files changed, 46 insertions(+), 28 deletions(-) diff --git a/cmd/eksterd/hubbackend.go b/cmd/eksterd/hubbackend.go index 2a5817b..9a3fb57 100644 --- a/cmd/eksterd/hubbackend.go +++ b/cmd/eksterd/hubbackend.go @@ -22,7 +22,6 @@ import ( "io" "log" "net/http" - "os" "strconv" "strings" "time" @@ -51,6 +50,17 @@ type hubIncomingBackend struct { baseURL string } +type Feed struct { + ID int64 `redis:"id"` + Channel string `redis:"channel"` + URL string `redis:"url"` + Callback string `redis:"callback"` + Hub string `redis:"hub"` + Secret string `redis:"secret"` + LeaseSeconds int64 `redis:"lease_seconds"` + ResubscribeAt int64 `redis:"resubscribe_at"` +} + func (h *hubIncomingBackend) GetSecret(id int64) string { conn := pool.Get() defer conn.Close() @@ -66,7 +76,6 @@ func (h *hubIncomingBackend) CreateFeed(topic string, channel string) (int64, er defer conn.Close() // TODO(peter): check if topic already is registered - id, err := redis.Int64(conn.Do("INCR", "feed:next_id")) if err != nil { return 0, err @@ -80,12 +89,13 @@ func (h *hubIncomingBackend) CreateFeed(topic string, channel string) (int64, er client := &http.Client{} hubURL, err := websub.GetHubURL(client, topic) - if hubURL == "" { + if err != nil { log.Printf("WebSub Hub URL not found for topic=%s\n", topic) - } else { - log.Printf("WebSub Hub URL found for topic=%s hub=%s\n", topic, hubURL) + return 0, err } + log.Printf("WebSub Hub URL found for topic=%s hub=%s\n", topic, hubURL) + callbackURL := fmt.Sprintf("%s/incoming/%d", h.baseURL, id) if err == nil && hubURL != "" { @@ -137,17 +147,6 @@ func (h *hubIncomingBackend) FeedSetLeaseSeconds(feedID int64, leaseSeconds int6 return nil } -type Feed struct { - ID int64 `redis:"id"` - Channel string `redis:"channel"` - URL string `redis:"url"` - Callback string `redis:"callback"` - Hub string `redis:"hub"` - Secret string `redis:"secret"` - LeaseSeconds int64 `redis:"lease_seconds"` - ResubscribeAt int64 `redis:"resubscribe_at"` -} - func (h *hubIncomingBackend) GetFeeds() []Feed { conn := pool.Get() defer conn.Close() @@ -209,10 +208,13 @@ func (h *hubIncomingBackend) run() error { log.Printf("Looking at %s\n", feed.URL) if feed.ResubscribeAt == 0 || time.Now().After(time.Unix(feed.ResubscribeAt, 0)) { if feed.Callback == "" { - feed.Callback = fmt.Sprintf("%s/incoming/%d", os.Getenv("EKSTER_BASEURL"), feed.ID) + feed.Callback = fmt.Sprintf("%s/incoming/%d", h.baseURL, feed.ID) } log.Printf("Send resubscribe for %s\n", feed.URL) - h.Subscribe(&feed) + err := h.Subscribe(&feed) + if err != nil { + log.Printf("Error while subscribing: %s", err) + } } } case <-quit: diff --git a/pkg/fetch/fetch.go b/pkg/fetch/fetch.go index 8bcb7bb..cc60700 100644 --- a/pkg/fetch/fetch.go +++ b/pkg/fetch/fetch.go @@ -72,9 +72,7 @@ func FeedHeader(fetcher Fetcher, fetchURL, contentType string, body io.Reader) ( feed.Name = author.Name feed.Photo = author.Photo } else if strings.HasPrefix(contentType, "application/json") { // json feed? - var jfeed jsonfeed.Feed - dec := json.NewDecoder(body) - err := dec.Decode(&jfeed) + jfeed, err := jsonfeed.Parse(body) if err != nil { log.Printf("Error while parsing json feed: %s\n", err) return feed, err diff --git a/pkg/jf2/simplify.go b/pkg/jf2/simplify.go index 4e55567..fd6e866 100644 --- a/pkg/jf2/simplify.go +++ b/pkg/jf2/simplify.go @@ -96,8 +96,7 @@ func simplifyToItem(itemType string, item map[string][]interface{}) microsub.Ite case "bookmark-of", "like-of", "repost-of", "in-reply-to": u, withItem, refItem := simplifyRefItem(k, v) - resultPtr := itemPtr(&feedItem, k) - if resultPtr != nil { + if resultPtr := itemPtr(&feedItem, k); resultPtr != nil { *resultPtr = append(*resultPtr, u) if withItem { feedItem.Refs[u] = refItem @@ -113,17 +112,23 @@ func simplifyToItem(itemType string, item map[string][]interface{}) microsub.Ite author, _ := simplifyCard(v[0]) feedItem.Checkin = &author case "name", "published", "updated", "url", "uid", "latitude", "longitude": - resultPtr := getScalarPtr(&feedItem, k) - if resultPtr != nil { + if resultPtr := getScalarPtr(&feedItem, k); resultPtr != nil { if len(v) >= 1 { *resultPtr = v[0].(string) } } case "category": - resultPtr := itemPtr(&feedItem, k) - if resultPtr != nil { + if resultPtr := itemPtr(&feedItem, k); resultPtr != nil { for _, c := range v { - *resultPtr = append(*resultPtr, c.(string)) + switch t := c.(type) { + case microformats.Microformat: + // TODO: perhaps use name + if t.Value != "" { + *resultPtr = append(*resultPtr, t.Value) + } + case string: + *resultPtr = append(*resultPtr, t) + } } } default: diff --git a/pkg/jsonfeed/jsonfeed.go b/pkg/jsonfeed/jsonfeed.go index 2a999a2..27d9de4 100644 --- a/pkg/jsonfeed/jsonfeed.go +++ b/pkg/jsonfeed/jsonfeed.go @@ -17,6 +17,11 @@ */ package jsonfeed +import ( + "encoding/json" + "io" +) + type Attachment struct { URL string `json:"url"` MimeType string `json:"mime_type"` @@ -63,3 +68,11 @@ type Feed struct { Items []Item `json:"items"` Hubs []Hub `json:"hubs"` } + +// Parse parses a jsonfeed +func Parse(body io.Reader) (Feed, error) { + var feed Feed + dec := json.NewDecoder(body) + err := dec.Decode(&feed) + return feed, err +}