peter/ekster
1
0
Fork 0
Code Issues 3 Pull Requests Projects 3 Releases 1 Wiki Activity

Gather more information from the right feed, check content-type

Browse Source
This commit is contained in:
Peter Stuifzand 2018-04-08 13:19:04 +02:00
parent 287bb3c3ed
commit 8302ecf3ce
2 changed files with 33 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
cmd/server/fetch.go
View File

@ -115,23 +115,34 @@ func (b *memoryBackend) feedHeader(fetchURL, contentType string, body io.Reader)
} else if strings.HasPrefix(contentType, "application/json") { // json feed? } else if strings.HasPrefix(contentType, "application/json") { // json feed?
var jfeed JSONFeed var jfeed JSONFeed
dec := json.NewDecoder(body) dec := json.NewDecoder(body)
err := dec.Decode(&feed) err := dec.Decode(&jfeed)
if err != nil { if err != nil {
log.Printf("Error while parsing json feed: %s\n", err) log.Printf("Error while parsing json feed: %s\n", err)
return feed, err return feed, err
} }
feed.Type = "feed" feed.Type = "feed"
feed.Name = jfeed.Author.Name feed.Name = jfeed.Title
if feed.Name == "" {
feed.Name = jfeed.Author.Name
}
feed.URL = jfeed.FeedURL feed.URL = jfeed.FeedURL
if feed.URL == "" { if feed.URL == "" {
feed.URL = fetchURL feed.URL = fetchURL
} }
feed.Photo = jfeed.Author.Avatar feed.Photo = jfeed.Icon
if feed.Photo == "" { if feed.Photo == "" {
feed.Photo = jfeed.Icon feed.Photo = jfeed.Author.Avatar
} }
feed.Author.Type = "card"
feed.Author.Name = jfeed.Author.Name
feed.Author.URL = jfeed.Author.URL
feed.Author.Photo = jfeed.Author.Avatar
return feed, nil return feed, nil
} else if strings.HasPrefix(contentType, "text/xml") || strings.HasPrefix(contentType, "application/rss+xml") || strings.HasPrefix(contentType, "application/atom+xml") || strings.HasPrefix(contentType, "application/xml") { } else if strings.HasPrefix(contentType, "text/xml") || strings.HasPrefix(contentType, "application/rss+xml") || strings.HasPrefix(contentType, "application/atom+xml") || strings.HasPrefix(contentType, "application/xml") {
@ -145,6 +156,7 @@ func (b *memoryBackend) feedHeader(fetchURL, contentType string, body io.Reader)
log.Printf("Error while parsing rss/atom feed: %s\n", err) log.Printf("Error while parsing rss/atom feed: %s\n", err)
return feed, err return feed, err
} }
log.Printf("%#v\n", xfeed)
feed.Type = "feed" feed.Type = "feed"
feed.Name = xfeed.Title feed.Name = xfeed.Title
@ -403,6 +415,9 @@ func Fetch2(fetchURL string) (*http.Response, error) {
cachedCopy := make([]byte, b.Len()) cachedCopy := make([]byte, b.Len())
cur := b.Bytes() cur := b.Bytes()
copy(cachedCopy, cur) copy(cachedCopy, cur)
log.Println(string(cachedCopy))
cache[u.String()] = cacheItem{item: cachedCopy, created: time.Now()} cache[u.String()] = cacheItem{item: cachedCopy, created: time.Now()}
cachedResp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(cachedCopy)), req) cachedResp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(cachedCopy)), req)

26
cmd/server/memory.go
View File

@ -450,20 +450,22 @@ func (b *memoryBackend) Search(query string) []microsub.Feed {
relURL := md.RelURLs[alt] relURL := md.RelURLs[alt]
log.Printf("alternate found with type %s %#v\n", relURL.Type, relURL) log.Printf("alternate found with type %s %#v\n", relURL.Type, relURL)
feedResp, err := Fetch2(alt) if strings.HasPrefix(relURL.Type, "text/html") || strings.HasPrefix(relURL.Type, "application/json") || strings.HasPrefix(relURL.Type, "application/xml") || strings.HasPrefix(relURL.Type, "text/xml") || strings.HasPrefix(relURL.Type, "application/rss+xml") || strings.HasPrefix(relURL.Type, "application/atom+xml") {
if err != nil { feedResp, err := Fetch2(alt)
log.Printf("Error in fetch of %s - %v\n", alt, err) if err != nil {
continue log.Printf("Error in fetch of %s - %v\n", alt, err)
} continue
defer feedResp.Body.Close() }
defer feedResp.Body.Close()
parsedFeed, err := b.feedHeader(alt, feedResp.Header.Get("Content-Type"), feedResp.Body) parsedFeed, err := b.feedHeader(alt, feedResp.Header.Get("Content-Type"), feedResp.Body)
if err != nil { if err != nil {
log.Printf("Error in parse of %s - %v\n", alt, err) log.Printf("Error in parse of %s - %v\n", alt, err)
continue continue
} }
feeds = append(feeds, parsedFeed) feeds = append(feeds, parsedFeed)
}
} }
} }
} }