peter/ekster
1
0
Fork 0
Code Issues 3 Pull Requests Projects 3 Releases 1 Wiki Activity

Improve handling of missing author and content

Browse Source
This commit is contained in:
Peter Stuifzand 2018-04-11 18:47:57 +02:00
parent 0a7696ea6b
commit 999bc0d456
3 changed files with 161 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

285
cmd/server/fetch.go
View File

@ -29,7 +29,6 @@ import (
"log" "log"
"net/http" "net/http"
"net/url" "net/url"
"os"
"rss" "rss"
"strings" "strings"
"time" "time"
@ -259,7 +258,7 @@ func (b *memoryBackend) feedItems(fetchURL, contentType string, body io.Reader)
log.Printf("%#v\n", feed) log.Printf("%#v\n", feed)
author := microsub.Card{} author := &microsub.Card{}
author.Type = "card" author.Type = "card"
author.Name = feed.Author.Name author.Name = feed.Author.Name
author.URL = feed.Author.URL author.URL = feed.Author.URL
@ -273,6 +272,7 @@ func (b *memoryBackend) feedItems(fetchURL, contentType string, body io.Reader)
var item microsub.Item var item microsub.Item
item.Type = "entry" item.Type = "entry"
item.Name = feedItem.Title item.Name = feedItem.Title
item.Content = &microsub.Content{}
item.Content.HTML = feedItem.ContentHTML item.Content.HTML = feedItem.ContentHTML
item.Content.Text = feedItem.ContentText item.Content.Text = feedItem.ContentText
item.URL = feedItem.URL item.URL = feedItem.URL
@ -280,7 +280,7 @@ func (b *memoryBackend) feedItems(fetchURL, contentType string, body io.Reader)
item.ID = hex.EncodeToString([]byte(feedItem.ID)) item.ID = hex.EncodeToString([]byte(feedItem.ID))
item.Published = feedItem.DatePublished item.Published = feedItem.DatePublished
itemAuthor := microsub.Card{} itemAuthor := &microsub.Card{}
itemAuthor.Type = "card" itemAuthor.Type = "card"
itemAuthor.Name = feedItem.Author.Name itemAuthor.Name = feedItem.Author.Name
itemAuthor.URL = feedItem.Author.URL itemAuthor.URL = feedItem.Author.URL
@ -309,6 +309,7 @@ func (b *memoryBackend) feedItems(fetchURL, contentType string, body io.Reader)
var item microsub.Item var item microsub.Item
item.Type = "entry" item.Type = "entry"
item.Name = feedItem.Title item.Name = feedItem.Title
item.Content = &microsub.Content{}
if len(feedItem.Content) > 0 { if len(feedItem.Content) > 0 {
item.Content.HTML = feedItem.Content item.Content.HTML = feedItem.Content
} else if len(feedItem.Summary) > 0 { } else if len(feedItem.Summary) > 0 {
@ -481,156 +482,156 @@ func Fetch2(fetchURL string) (*http.Response, error) {
return cachedResp, err return cachedResp, err
} }
func Fetch(fetchURL string) []microsub.Item { // func Fetch(fetchURL string) []microsub.Item {
result := []microsub.Item{} // result := []microsub.Item{}
if !strings.HasPrefix(fetchURL, "http") { // if !strings.HasPrefix(fetchURL, "http") {
return result // return result
} // }
u, err := url.Parse(fetchURL) // u, err := url.Parse(fetchURL)
if err != nil { // if err != nil {
log.Printf("error parsing %s as url: %s", fetchURL, err) // log.Printf("error parsing %s as url: %s", fetchURL, err)
return result // return result
} // }
resp, err := http.Get(u.String()) // resp, err := http.Get(u.String())
if err != nil { // if err != nil {
log.Printf("error while fetching %s: %s", u, err) // log.Printf("error while fetching %s: %s", u, err)
return result // return result
} // }
if !strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") { // if !strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") {
log.Printf("Content Type of %s = %s", fetchURL, resp.Header.Get("Content-Type")) // log.Printf("Content Type of %s = %s", fetchURL, resp.Header.Get("Content-Type"))
return result // return result
} // }
defer resp.Body.Close() // defer resp.Body.Close()
data := microformats.Parse(resp.Body, u) // data := microformats.Parse(resp.Body, u)
jw := json.NewEncoder(os.Stdout) // jw := json.NewEncoder(os.Stdout)
jw.SetIndent("", " ") // jw.SetIndent("", " ")
jw.Encode(data) // jw.Encode(data)
author := microsub.Card{} // author := &microsub.Card{}
for _, item := range data.Items { // for _, item := range data.Items {
if item.Type[0] == "h-feed" { // if item.Type[0] == "h-feed" {
for _, child := range item.Children { // for _, child := range item.Children {
previewItem := convertMfToItem(child) // previewItem := convertMfToItem(child)
result = append(result, previewItem) // result = append(result, previewItem)
} // }
} else if item.Type[0] == "h-card" { // } else if item.Type[0] == "h-card" {
mf := item // mf := item
author.Filled = true // author.Filled = true
author.Type = "card" // author.Type = "card"
for prop, value := range mf.Properties { // for prop, value := range mf.Properties {
switch prop { // switch prop {
case "url": // case "url":
author.URL = value[0].(string) // author.URL = value[0].(string)
break // break
case "name": // case "name":
author.Name = value[0].(string) // author.Name = value[0].(string)
break // break
case "photo": // case "photo":
author.Photo = value[0].(string) // author.Photo = value[0].(string)
break // break
default: // default:
fmt.Printf("prop name not implemented for author: %s with value %#v\n", prop, value) // fmt.Printf("prop name not implemented for author: %s with value %#v\n", prop, value)
break // break
} // }
} // }
} else if item.Type[0] == "h-entry" { // } else if item.Type[0] == "h-entry" {
previewItem := convertMfToItem(item) // previewItem := convertMfToItem(item)
result = append(result, previewItem) // result = append(result, previewItem)
} // }
} // }
for i, item := range result { // for i, item := range result {
if !item.Author.Filled { // if !item.Author.Filled {
result[i].Author = author // result[i].Author = author
} // }
} // }
return result // return result
} // }
func convertMfToItem(mf *microformats.Microformat) microsub.Item { // func convertMfToItem(mf *microformats.Microformat) microsub.Item {
item := microsub.Item{} // item := microsub.Item{}
item.Type = mf.Type[0] // item.Type = mf.Type[0]
for prop, value := range mf.Properties { // for prop, value := range mf.Properties {
switch prop { // switch prop {
case "published": // case "published":
item.Published = value[0].(string) // item.Published = value[0].(string)
break // break
case "url": // case "url":
item.URL = value[0].(string) // item.URL = value[0].(string)
break // break
case "name": // case "name":
item.Name = value[0].(string) // item.Name = value[0].(string)
break // break
case "latitude": // case "latitude":
item.Latitude = value[0].(string) // item.Latitude = value[0].(string)
break // break
case "longitude": // case "longitude":
item.Longitude = value[0].(string) // item.Longitude = value[0].(string)
break // break
case "like-of": // case "like-of":
for _, v := range value { // for _, v := range value {
item.LikeOf = append(item.LikeOf, v.(string)) // item.LikeOf = append(item.LikeOf, v.(string))
} // }
break // break
case "bookmark-of": // case "bookmark-of":
for _, v := range value { // for _, v := range value {
item.BookmarkOf = append(item.BookmarkOf, v.(string)) // item.BookmarkOf = append(item.BookmarkOf, v.(string))
} // }
break // break
case "in-reply-to": // case "in-reply-to":
for _, v := range value { // for _, v := range value {
item.InReplyTo = append(item.InReplyTo, v.(string)) // item.InReplyTo = append(item.InReplyTo, v.(string))
} // }
break // break
case "summary": // case "summary":
if content, ok := value[0].(map[string]interface{}); ok { // if content, ok := value[0].(map[string]interface{}); ok {
item.Content.HTML = content["html"].(string) // item.Content.HTML = content["html"].(string)
item.Content.Text = content["value"].(string) // item.Content.Text = content["value"].(string)
} else if content, ok := value[0].(string); ok { // } else if content, ok := value[0].(string); ok {
item.Content.Text = content // item.Content.Text = content
} // }
break // break
case "photo": // case "photo":
for _, v := range value { // for _, v := range value {
item.Photo = append(item.Photo, v.(string)) // item.Photo = append(item.Photo, v.(string))
} // }
break // break
case "category": // case "category":
for _, v := range value { // for _, v := range value {
item.Category = append(item.Category, v.(string)) // item.Category = append(item.Category, v.(string))
} // }
break // break
case "content": // case "content":
if content, ok := value[0].(map[string]interface{}); ok { // if content, ok := value[0].(map[string]interface{}); ok {
item.Content.HTML = content["html"].(string) // item.Content.HTML = content["html"].(string)
item.Content.Text = content["value"].(string) // item.Content.Text = content["value"].(string)
} else if content, ok := value[0].(string); ok { // } else if content, ok := value[0].(string); ok {
item.Content.Text = content // item.Content.Text = content
} // }
break // break
default: // default:
fmt.Printf("prop name not implemented: %s with value %#v\n", prop, value) // fmt.Printf("prop name not implemented: %s with value %#v\n", prop, value)
break // break
} // }
} // }
if item.Name == strings.TrimSpace(item.Content.Text) { // if item.Name == strings.TrimSpace(item.Content.Text) {
item.Name = "" // item.Name = ""
} // }
// TODO: for like name is the field that is set // // TODO: for like name is the field that is set
if item.Content.HTML == "" && len(item.LikeOf) > 0 { // if item.Content.HTML == "" && len(item.LikeOf) > 0 {
item.Name = "" // item.Name = ""
} // }
fmt.Printf("%#v\n", item) // fmt.Printf("%#v\n", item)
return item // return item
} // }

15
cmd/server/memory.go
View File

@ -183,8 +183,8 @@ func (b *memoryBackend) ChannelsDelete(uid string) {
} }
} }
func mapToAuthor(result map[string]string) microsub.Card { func mapToAuthor(result map[string]string) *microsub.Card {
item := microsub.Card{} item := &microsub.Card{}
item.Type = "card" item.Type = "card"
if name, e := result["name"]; e { if name, e := result["name"]; e {
item.Name = name item.Name = name
@ -236,14 +236,21 @@ func mapToItem(result map[string]interface{}) microsub.Item {
} }
if content, e := result["content"]; e { if content, e := result["content"]; e {
itemContent := &microsub.Content{}
set := false
if c, ok := content.(map[string]interface{}); ok { if c, ok := content.(map[string]interface{}); ok {
if html, e2 := c["html"]; e2 { if html, e2 := c["html"]; e2 {
item.Content.HTML = html.(string) itemContent.HTML = html.(string)
set = true
} }
if text, e2 := c["value"]; e2 { if text, e2 := c["value"]; e2 {
item.Content.Text = text.(string) itemContent.Text = text.(string)
set = true
} }
} }
if set {
item.Content = itemContent
}
} }
// TODO: Check how to improve this // TODO: Check how to improve this

14
microsub/protocol.go
View File

@ -59,11 +59,11 @@ type Content struct {
type Item struct { type Item struct {
Type string `json:"type"` Type string `json:"type"`
Name string `json:"name,omitempty"` Name string `json:"name,omitempty"`
Published string `json:"published"` Published string `json:"published,omitempty"`
Updated string `json:"updated"` Updated string `json:"updated,omitempty"`
URL string `json:"url"` URL string `json:"url,omitempty"`
UID string `json:"uid,omitempty"` UID string `json:"uid,omitempty"`
Author Card `json:"author,omitempty"` Author *Card `json:"author,omitempty"`
Category []string `json:"category,omitempty"` Category []string `json:"category,omitempty"`
Photo []string `json:"photo,omitempty"` Photo []string `json:"photo,omitempty"`
LikeOf []string `json:"like-of,omitempty"` LikeOf []string `json:"like-of,omitempty"`
@ -71,11 +71,11 @@ type Item struct {
RepostOf []string `json:"repost-of,omitempty"` RepostOf []string `json:"repost-of,omitempty"`
InReplyTo []string `json:"in-reply-to,omitempty"` InReplyTo []string `json:"in-reply-to,omitempty"`
Summary []string `json:"summary,omitempty"` Summary []string `json:"summary,omitempty"`
Content Content `json:"content,omitempty"` Content *Content `json:"content,omitempty"`
Latitude string `json:"latitude,omitempty"` Latitude string `json:"latitude,omitempty"`
Longitude string `json:"longitude,omitempty"` Longitude string `json:"longitude,omitempty"`
Checkin Card `json:"checkin,omitempty"` Checkin *Card `json:"checkin,omitempty"`
ID string `json:"_id"` ID string `json:"_id,omitempty"`
Read bool `json:"_is_read"` Read bool `json:"_is_read"`
} }