peter/ekster
1
0
Fork 0
Code Issues 3 Pull Requests Projects 3 Releases 1 Wiki Activity

Move fetching code to fetch package

Browse Source
This commit is contained in:
Peter Stuifzand 2018-08-05 13:45:12 +02:00
parent 573816d75f
commit 1cb3e21e7c
Signed by: peter
GPG Key ID: 374322D56E5209E8
5 changed files with 244 additions and 232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

234
cmd/eksterd/memory.go
View File

@ -18,6 +18,8 @@
package main
import (
"bufio"
"bytes"
"encoding/json"
"fmt"
"io"
@ -31,6 +33,7 @@ import (
"time"
"p83.nl/go/ekster/pkg/feedbin"
"p83.nl/go/ekster/pkg/fetch"
"p83.nl/go/ekster/pkg/microsub"
"github.com/garyburd/redigo/redis"
@ -58,6 +61,18 @@ type Debug interface {
Debug()
}
type redisItem struct {
ID string
Published string
Read bool
Data []byte
}
type fetch2 struct {}
func (f *fetch2) Fetch(url string) (*http.Response, error) {
return Fetch2(url)
}
func (b *memoryBackend) Debug() {
fmt.Println(b.Channels)
fmt.Println(b.Feeds)
@ -87,7 +102,7 @@ func (b *memoryBackend) load() error {
for uid, channel := range b.Channels {
log.Printf("loading channel %s - %s\n", uid, channel.Name)
// for _, feed := range b.Feeds[uid] {
//log.Printf("- loading feed %s\n", feed.URL)
// log.Printf("- loading feed %s\n", feed.URL)
// resp, err := b.Fetch3(uid, feed.URL)
// if err != nil {
// log.Printf("Error while Fetch3 of %s: %v\n", feed.URL, err)
@ -217,164 +232,6 @@ func (b *memoryBackend) ChannelsDelete(uid string) error {
return nil
}
func mapToAuthor(result map[string]string) *microsub.Card {
item := &microsub.Card{}
item.Type = "card"
if name, e := result["name"]; e {
item.Name = name
}
if u, e := result["url"]; e {
item.URL = u
}
if photo, e := result["photo"]; e {
item.Photo = photo
}
if value, e := result["longitude"]; e {
item.Longitude = value
}
if value, e := result["latitude"]; e {
item.Latitude = value
}
if value, e := result["country-name"]; e {
item.CountryName = value
}
if value, e := result["locality"]; e {
item.Locality = value
}
return item
}
func mapToItem(result map[string]interface{}) microsub.Item {
item := microsub.Item{}
item.Type = "entry"
if name, e := result["name"]; e {
item.Name = name.(string)
}
if url, e := result["url"]; e {
item.URL = url.(string)
}
if uid, e := result["uid"]; e {
item.UID = uid.(string)
}
if author, e := result["author"]; e {
item.Author = mapToAuthor(author.(map[string]string))
}
if checkin, e := result["checkin"]; e {
item.Checkin = mapToAuthor(checkin.(map[string]string))
}
if content, e := result["content"]; e {
itemContent := &microsub.Content{}
set := false
if c, ok := content.(map[string]interface{}); ok {
if html, e2 := c["html"]; e2 {
itemContent.HTML = html.(string)
set = true
}
if text, e2 := c["value"]; e2 {
itemContent.Text = text.(string)
set = true
}
}
if set {
item.Content = itemContent
}
}
// TODO: Check how to improve this
if value, e := result["like-of"]; e {
for _, v := range value.([]interface{}) {
if u, ok := v.(string); ok {
item.LikeOf = append(item.LikeOf, u)
}
}
}
if value, e := result["repost-of"]; e {
if repost, ok := value.(string); ok {
item.RepostOf = append(item.RepostOf, repost)
} else if repost, ok := value.([]interface{}); ok {
for _, v := range repost {
if u, ok := v.(string); ok {
item.RepostOf = append(item.RepostOf, u)
}
}
}
}
if value, e := result["bookmark-of"]; e {
for _, v := range value.([]interface{}) {
if u, ok := v.(string); ok {
item.BookmarkOf = append(item.BookmarkOf, u)
}
}
}
if value, e := result["in-reply-to"]; e {
if replyTo, ok := value.(string); ok {
item.InReplyTo = append(item.InReplyTo, replyTo)
} else if valueArray, ok := value.([]interface{}); ok {
for _, v := range valueArray {
if replyTo, ok := v.(string); ok {
item.InReplyTo = append(item.InReplyTo, replyTo)
} else if cite, ok := v.(map[string]interface{}); ok {
item.InReplyTo = append(item.InReplyTo, cite["url"].(string))
}
}
}
}
if value, e := result["photo"]; e {
for _, v := range value.([]interface{}) {
item.Photo = append(item.Photo, v.(string))
}
}
if value, e := result["category"]; e {
if cats, ok := value.([]string); ok {
for _, v := range cats {
item.Category = append(item.Category, v)
}
} else if cats, ok := value.([]interface{}); ok {
for _, v := range cats {
if cat, ok := v.(string); ok {
item.Category = append(item.Category, cat)
} else if cat, ok := v.(map[string]interface{}); ok {
item.Category = append(item.Category, cat["value"].(string))
}
}
} else if cat, ok := value.(string); ok {
item.Category = append(item.Category, cat)
}
}
if published, e := result["published"]; e {
item.Published = published.(string)
} else {
item.Published = time.Now().Format(time.RFC3339)
}
if updated, e := result["updated"]; e {
item.Updated = updated.(string)
}
if id, e := result["_id"]; e {
item.ID = id.(string)
}
if read, e := result["_is_read"]; e {
item.Read = read.(bool)
}
return item
}
func (b *memoryBackend) run() {
b.ticker = time.NewTicker(10 * time.Minute)
b.quit = make(chan struct{})
@ -454,9 +311,9 @@ func (b *memoryBackend) TimelineGet(before, after, channel string) (microsub.Tim
items := []microsub.Item{}
zchannelKey := fmt.Sprintf("zchannel:%s:posts", channel)
//channelKey := fmt.Sprintf("channel:%s:posts", channel)
// channelKey := fmt.Sprintf("channel:%s:posts", channel)
//itemJsons, err := redis.ByteSlices(conn.Do("SORT", channelKey, "BY", "*->Published", "GET", "*->Data", "ASC", "ALPHA"))
// itemJsons, err := redis.ByteSlices(conn.Do("SORT", channelKey, "BY", "*->Published", "GET", "*->Data", "ASC", "ALPHA"))
// if err != nil {
// log.Println(err)
// return microsub.Timeline{
@ -533,7 +390,7 @@ func (b *memoryBackend) TimelineGet(before, after, channel string) (microsub.Tim
}, nil
}
//panic if s is not a slice
// panic if s is not a slice
func reverseSlice(s interface{}) {
size := reflect.ValueOf(s).Len()
swap := reflect.Swapper(s)
@ -667,7 +524,7 @@ func (b *memoryBackend) Search(query string) ([]microsub.Feed, error) {
}
defer feedResp.Body.Close()
parsedFeed, err := feedHeader(fetchUrl.String(), feedResp.Header.Get("Content-Type"), feedResp.Body)
parsedFeed, err := fetch.FeedHeader(&fetch2{}, fetchUrl.String(), feedResp.Header.Get("Content-Type"), feedResp.Body)
if err != nil {
log.Printf("Error in parse of %s - %v\n", fetchUrl, err)
continue
@ -689,7 +546,7 @@ func (b *memoryBackend) Search(query string) ([]microsub.Feed, error) {
// FIXME: don't defer in for loop (possible memory leak)
defer feedResp.Body.Close()
parsedFeed, err := feedHeader(alt, feedResp.Header.Get("Content-Type"), feedResp.Body)
parsedFeed, err := fetch.FeedHeader(&fetch2{}, alt, feedResp.Header.Get("Content-Type"), feedResp.Body)
if err != nil {
log.Printf("Error in parse of %s - %v\n", alt, err)
continue
@ -709,7 +566,7 @@ func (b *memoryBackend) PreviewURL(previewURL string) (microsub.Timeline, error)
if err != nil {
return microsub.Timeline{}, fmt.Errorf("error while fetching %s: %v", previewURL, err)
}
items, err := feedItems(previewURL, resp.Header.Get("content-type"), resp.Body)
items, err := fetch.FeedItems(&fetch2{}, previewURL, resp.Header.Get("content-type"), resp.Body)
if err != nil {
return microsub.Timeline{}, fmt.Errorf("error while fetching %s: %v", previewURL, err)
}
@ -756,7 +613,7 @@ func (b *memoryBackend) ProcessContent(channel, fetchURL, contentType string, bo
conn := pool.Get()
defer conn.Close()
items, err := feedItems(fetchURL, contentType, body)
items, err := fetch.FeedItems(&fetch2{}, fetchURL, contentType, body)
if err != nil {
return err
}
@ -893,3 +750,48 @@ func (b *memoryBackend) updateChannelUnreadCount(conn redis.Conn, channel string
return nil
}
// Fetch2 fetches stuff
func Fetch2(fetchURL string) (*http.Response, error) {
conn := pool.Get()
defer conn.Close()
if !strings.HasPrefix(fetchURL, "http") {
return nil, fmt.Errorf("error parsing %s as url, has no http(s) prefix", fetchURL)
}
u, err := url.Parse(fetchURL)
if err != nil {
return nil, fmt.Errorf("error parsing %s as url: %s", fetchURL, err)
}
req, err := http.NewRequest("GET", u.String(), nil)
cacheKey := fmt.Sprintf("http_cache:%s", u.String())
data, err := redis.Bytes(conn.Do("GET", cacheKey))
if err == nil {
log.Printf("HIT %s\n", u.String())
rd := bufio.NewReader(bytes.NewReader(data))
return http.ReadResponse(rd, req)
}
log.Printf("MISS %s\n", u.String())
client := http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("error while fetching %s: %s", u, err)
}
defer resp.Body.Close()
var b bytes.Buffer
resp.Write(&b)
cachedCopy := make([]byte, b.Len())
cur := b.Bytes()
copy(cachedCopy, cur)
conn.Do("SET", cacheKey, cachedCopy, "EX", 60*60)
cachedResp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(cachedCopy)), req)
return cachedResp, err
}

2
cmd/eksterd/micropub.go
View File

@ -92,7 +92,7 @@ func (h *micropubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return
}
item = mapToItem(jf2.SimplifyMicroformat(&mfItem))
item = jf2.MapToItem(jf2.SimplifyMicroformat(&mfItem))
ok = true
} else if r.Header.Get("Content-Type") == "application/x-www-form-urlencoded" {
content := r.FormValue("content")

72
cmd/eksterd/fetch.go → pkg/fetch/fetch.go
View File

@ -16,18 +16,14 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package main
package fetch
import (
"bufio"
"bytes"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"net/url"
"strings"
"time"
@ -38,11 +34,10 @@ import (
"p83.nl/go/ekster/pkg/jsonfeed"
"p83.nl/go/ekster/pkg/microsub"
"github.com/garyburd/redigo/redis"
"willnorris.com/go/microformats"
)
func feedHeader(fetchURL, contentType string, body io.Reader) (microsub.Feed, error) {
func FeedHeader(fetcher Fetcher, fetchURL, contentType string, body io.Reader) (microsub.Feed, error) {
log.Printf("ProcessContent %s\n", fetchURL)
log.Println("Found " + contentType)
@ -68,7 +63,7 @@ func feedHeader(fetchURL, contentType string, body io.Reader) (microsub.Feed, er
if as, ok := card.(string); ok {
if strings.HasPrefix(as, "http") {
resp, err := Fetch2(fetchURL)
resp, err := fetcher.Fetch(fetchURL)
if err != nil {
return feed, err
}
@ -155,7 +150,7 @@ func feedHeader(fetchURL, contentType string, body io.Reader) (microsub.Feed, er
return feed, nil
}
func feedItems(fetchURL, contentType string, body io.Reader) ([]microsub.Item, error) {
func FeedItems(fetcher Fetcher, fetchURL, contentType string, body io.Reader) ([]microsub.Item, error) {
log.Printf("ProcessContent %s\n", fetchURL)
log.Println("Found " + contentType)
@ -191,7 +186,7 @@ func feedItems(fetchURL, contentType string, body io.Reader) ([]microsub.Item, e
for i, r := range results {
if as, ok := r["author"].(string); ok {
if r["type"] == "entry" && strings.HasPrefix(as, "http") {
resp, err := Fetch2(fetchURL)
resp, err := fetcher.Fetch(fetchURL)
if err != nil {
return items, err
}
@ -218,11 +213,11 @@ func feedItems(fetchURL, contentType string, body io.Reader) ([]microsub.Item, e
r["_id"] = hex.EncodeToString([]byte(uid.(string)))
} else {
continue
//r["_id"] = "" // generate random value
// r["_id"] = "" // generate random value
}
// mapToItem adds published
item := mapToItem(r)
item := jf2.MapToItem(r)
items = append(items, item)
}
} else if strings.HasPrefix(contentType, "application/json") { // json feed?
@ -333,56 +328,3 @@ func feedItems(fetchURL, contentType string, body io.Reader) ([]microsub.Item, e
return items, nil
}
type redisItem struct {
ID string
Published string
Read bool
Data []byte
}
// Fetch2 fetches stuff
func Fetch2(fetchURL string) (*http.Response, error) {
conn := pool.Get()
defer conn.Close()
if !strings.HasPrefix(fetchURL, "http") {
return nil, fmt.Errorf("error parsing %s as url, has no http(s) prefix", fetchURL)
}
u, err := url.Parse(fetchURL)
if err != nil {
return nil, fmt.Errorf("error parsing %s as url: %s", fetchURL, err)
}
req, err := http.NewRequest("GET", u.String(), nil)
cacheKey := fmt.Sprintf("http_cache:%s", u.String())
data, err := redis.Bytes(conn.Do("GET", cacheKey))
if err == nil {
log.Printf("HIT %s\n", u.String())
rd := bufio.NewReader(bytes.NewReader(data))
return http.ReadResponse(rd, req)
}
log.Printf("MISS %s\n", u.String())
client := http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("error while fetching %s: %s", u, err)
}
defer resp.Body.Close()
var b bytes.Buffer
resp.Write(&b)
cachedCopy := make([]byte, b.Len())
cur := b.Bytes()
copy(cachedCopy, cur)
conn.Do("SET", cacheKey, cachedCopy, "EX", 60*60)
cachedResp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(cachedCopy)), req)
return cachedResp, err
}

7
pkg/fetch/fetcher.go Normal file
View File

@ -0,0 +1,7 @@
package fetch
import "net/http"
type Fetcher interface {
Fetch(url string) (*http.Response, error)
}

161
pkg/jf2/simplify.go
View File

@ -21,6 +21,9 @@ import (
"fmt"
"log"
"strings"
"time"
"p83.nl/go/ekster/pkg/microsub"
"willnorris.com/go/microformats"
)
@ -155,3 +158,161 @@ func SimplifyMicroformatData(md *microformats.Data) []map[string]interface{} {
}
return items
}
func MapToAuthor(result map[string]string) *microsub.Card {
item := &microsub.Card{}
item.Type = "card"
if name, e := result["name"]; e {
item.Name = name
}
if u, e := result["url"]; e {
item.URL = u
}
if photo, e := result["photo"]; e {
item.Photo = photo
}
if value, e := result["longitude"]; e {
item.Longitude = value
}
if value, e := result["latitude"]; e {
item.Latitude = value
}
if value, e := result["country-name"]; e {
item.CountryName = value
}
if value, e := result["locality"]; e {
item.Locality = value
}
return item
}
func MapToItem(result map[string]interface{}) microsub.Item {
item := microsub.Item{}
item.Type = "entry"
if name, e := result["name"]; e {
item.Name = name.(string)
}
if url, e := result["url"]; e {
item.URL = url.(string)
}
if uid, e := result["uid"]; e {
item.UID = uid.(string)
}
if author, e := result["author"]; e {
item.Author = MapToAuthor(author.(map[string]string))
}
if checkin, e := result["checkin"]; e {
item.Checkin = MapToAuthor(checkin.(map[string]string))
}
if content, e := result["content"]; e {
itemContent := &microsub.Content{}
set := false
if c, ok := content.(map[string]interface{}); ok {
if html, e2 := c["html"]; e2 {
itemContent.HTML = html.(string)
set = true
}
if text, e2 := c["value"]; e2 {
itemContent.Text = text.(string)
set = true
}
}
if set {
item.Content = itemContent
}
}
// TODO: Check how to improve this
if value, e := result["like-of"]; e {
for _, v := range value.([]interface{}) {
if u, ok := v.(string); ok {
item.LikeOf = append(item.LikeOf, u)
}
}
}
if value, e := result["repost-of"]; e {
if repost, ok := value.(string); ok {
item.RepostOf = append(item.RepostOf, repost)
} else if repost, ok := value.([]interface{}); ok {
for _, v := range repost {
if u, ok := v.(string); ok {
item.RepostOf = append(item.RepostOf, u)
}
}
}
}
if value, e := result["bookmark-of"]; e {
for _, v := range value.([]interface{}) {
if u, ok := v.(string); ok {
item.BookmarkOf = append(item.BookmarkOf, u)
}
}
}
if value, e := result["in-reply-to"]; e {
if replyTo, ok := value.(string); ok {
item.InReplyTo = append(item.InReplyTo, replyTo)
} else if valueArray, ok := value.([]interface{}); ok {
for _, v := range valueArray {
if replyTo, ok := v.(string); ok {
item.InReplyTo = append(item.InReplyTo, replyTo)
} else if cite, ok := v.(map[string]interface{}); ok {
item.InReplyTo = append(item.InReplyTo, cite["url"].(string))
}
}
}
}
if value, e := result["photo"]; e {
for _, v := range value.([]interface{}) {
item.Photo = append(item.Photo, v.(string))
}
}
if value, e := result["category"]; e {
if cats, ok := value.([]string); ok {
for _, v := range cats {
item.Category = append(item.Category, v)
}
} else if cats, ok := value.([]interface{}); ok {
for _, v := range cats {
if cat, ok := v.(string); ok {
item.Category = append(item.Category, cat)
} else if cat, ok := v.(map[string]interface{}); ok {
item.Category = append(item.Category, cat["value"].(string))
}
}
} else if cat, ok := value.(string); ok {
item.Category = append(item.Category, cat)
}
}
if published, e := result["published"]; e {
item.Published = published.(string)
} else {
item.Published = time.Now().Format(time.RFC3339)
}
if updated, e := result["updated"]; e {
item.Updated = updated.(string)
}
if id, e := result["_id"]; e {
item.ID = id.(string)
}
if read, e := result["_is_read"]; e {
item.Read = read.(bool)
}
return item
}