Remove .idea files

.gitignore

@@ -1 +1,3 @@
-cmd/server/server
+cmd/eksterd/eksterd
+cmd/ek/ek
+.idea

.idea/codeStyles/codeStyleConfig.xml

@@ -1,5 +0,0 @@
-<component name="ProjectCodeStyleConfiguration">
-  <state>
-    <option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
-  </state>
-</component>

.idea/go.imports.xml

@@ -1,9 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="GoImports">
-    <option name="groupStdlibImports" value="true" />
-    <option name="importSorting" value="GOIMPORTS" />
-    <option name="moveAllImportsInOneDeclaration" value="true" />
-    <option name="moveAllStdlibImportsInOneGroup" value="true" />
-  </component>
-</project>

cmd/eksterd/memory.go

@@ -18,17 +18,22 @@
 package main
 
 import (
+	"bufio"
+	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
 	"log"
 	"net/http"
 	"net/url"
 	"os"
 	"reflect"
+	"regexp"
 	"strings"
 	"time"
 
 	"p83.nl/go/ekster/pkg/feedbin"
+	"p83.nl/go/ekster/pkg/fetch"
 	"p83.nl/go/ekster/pkg/microsub"
 
 	"github.com/garyburd/redigo/redis"
@@ -56,6 +61,18 @@ type Debug interface {
 	Debug()
 }
 
+type redisItem struct {
+	ID        string
+	Published string
+	Read      bool
+	Data      []byte
+}
+
+type fetch2 struct {}
+func (f *fetch2) Fetch(url string) (*http.Response, error) {
+	return Fetch2(url)
+}
+
 func (b *memoryBackend) Debug() {
 	fmt.Println(b.Channels)
 	fmt.Println(b.Feeds)
@@ -85,7 +102,7 @@ func (b *memoryBackend) load() error {
 	for uid, channel := range b.Channels {
 		log.Printf("loading channel %s - %s\n", uid, channel.Name)
 		// for _, feed := range b.Feeds[uid] {
-		//log.Printf("- loading feed %s\n", feed.URL)
+		// log.Printf("- loading feed %s\n", feed.URL)
 		// resp, err := b.Fetch3(uid, feed.URL)
 		// if err != nil {
 		// 	log.Printf("Error while Fetch3 of %s: %v\n", feed.URL, err)
@@ -215,164 +232,6 @@ func (b *memoryBackend) ChannelsDelete(uid string) error {
 	return nil
 }
 
-func mapToAuthor(result map[string]string) *microsub.Card {
-	item := &microsub.Card{}
-	item.Type = "card"
-	if name, e := result["name"]; e {
-		item.Name = name
-	}
-	if u, e := result["url"]; e {
-		item.URL = u
-	}
-	if photo, e := result["photo"]; e {
-		item.Photo = photo
-	}
-	if value, e := result["longitude"]; e {
-		item.Longitude = value
-	}
-	if value, e := result["latitude"]; e {
-		item.Latitude = value
-	}
-	if value, e := result["country-name"]; e {
-		item.CountryName = value
-	}
-	if value, e := result["locality"]; e {
-		item.Locality = value
-	}
-	return item
-}
-
-func mapToItem(result map[string]interface{}) microsub.Item {
-	item := microsub.Item{}
-
-	item.Type = "entry"
-
-	if name, e := result["name"]; e {
-		item.Name = name.(string)
-	}
-
-	if url, e := result["url"]; e {
-		item.URL = url.(string)
-	}
-
-	if uid, e := result["uid"]; e {
-		item.UID = uid.(string)
-	}
-
-	if author, e := result["author"]; e {
-		item.Author = mapToAuthor(author.(map[string]string))
-	}
-
-	if checkin, e := result["checkin"]; e {
-		item.Checkin = mapToAuthor(checkin.(map[string]string))
-	}
-
-	if content, e := result["content"]; e {
-		itemContent := &microsub.Content{}
-		set := false
-		if c, ok := content.(map[string]interface{}); ok {
-			if html, e2 := c["html"]; e2 {
-				itemContent.HTML = html.(string)
-				set = true
-			}
-			if text, e2 := c["value"]; e2 {
-				itemContent.Text = text.(string)
-				set = true
-			}
-		}
-		if set {
-			item.Content = itemContent
-		}
-	}
-
-	// TODO: Check how to improve this
-
-	if value, e := result["like-of"]; e {
-		for _, v := range value.([]interface{}) {
-			if u, ok := v.(string); ok {
-				item.LikeOf = append(item.LikeOf, u)
-			}
-		}
-	}
-
-	if value, e := result["repost-of"]; e {
-		if repost, ok := value.(string); ok {
-			item.RepostOf = append(item.RepostOf, repost)
-		} else if repost, ok := value.([]interface{}); ok {
-			for _, v := range repost {
-				if u, ok := v.(string); ok {
-					item.RepostOf = append(item.RepostOf, u)
-				}
-			}
-		}
-	}
-
-	if value, e := result["bookmark-of"]; e {
-		for _, v := range value.([]interface{}) {
-			if u, ok := v.(string); ok {
-				item.BookmarkOf = append(item.BookmarkOf, u)
-			}
-		}
-	}
-
-	if value, e := result["in-reply-to"]; e {
-		if replyTo, ok := value.(string); ok {
-			item.InReplyTo = append(item.InReplyTo, replyTo)
-		} else if valueArray, ok := value.([]interface{}); ok {
-			for _, v := range valueArray {
-				if replyTo, ok := v.(string); ok {
-					item.InReplyTo = append(item.InReplyTo, replyTo)
-				} else if cite, ok := v.(map[string]interface{}); ok {
-					item.InReplyTo = append(item.InReplyTo, cite["url"].(string))
-				}
-			}
-		}
-	}
-
-	if value, e := result["photo"]; e {
-		for _, v := range value.([]interface{}) {
-			item.Photo = append(item.Photo, v.(string))
-		}
-	}
-
-	if value, e := result["category"]; e {
-		if cats, ok := value.([]string); ok {
-			for _, v := range cats {
-				item.Category = append(item.Category, v)
-			}
-		} else if cats, ok := value.([]interface{}); ok {
-			for _, v := range cats {
-				if cat, ok := v.(string); ok {
-					item.Category = append(item.Category, cat)
-				} else if cat, ok := v.(map[string]interface{}); ok {
-					item.Category = append(item.Category, cat["value"].(string))
-				}
-			}
-		} else if cat, ok := value.(string); ok {
-			item.Category = append(item.Category, cat)
-		}
-	}
-
-	if published, e := result["published"]; e {
-		item.Published = published.(string)
-	} else {
-		item.Published = time.Now().Format(time.RFC3339)
-	}
-
-	if updated, e := result["updated"]; e {
-		item.Updated = updated.(string)
-	}
-
-	if id, e := result["_id"]; e {
-		item.ID = id.(string)
-	}
-	if read, e := result["_is_read"]; e {
-		item.Read = read.(bool)
-	}
-
-	return item
-}
-
 func (b *memoryBackend) run() {
 	b.ticker = time.NewTicker(10 * time.Minute)
 	b.quit = make(chan struct{})
@@ -452,9 +311,9 @@ func (b *memoryBackend) TimelineGet(before, after, channel string) (microsub.Tim
 	items := []microsub.Item{}
 
 	zchannelKey := fmt.Sprintf("zchannel:%s:posts", channel)
-	//channelKey := fmt.Sprintf("channel:%s:posts", channel)
+	// channelKey := fmt.Sprintf("channel:%s:posts", channel)
 
-	//itemJsons, err := redis.ByteSlices(conn.Do("SORT", channelKey, "BY", "*->Published", "GET", "*->Data", "ASC", "ALPHA"))
+	// itemJsons, err := redis.ByteSlices(conn.Do("SORT", channelKey, "BY", "*->Published", "GET", "*->Data", "ASC", "ALPHA"))
 	// if err != nil {
 	// 	log.Println(err)
 	// 	return microsub.Timeline{
@@ -531,7 +390,7 @@ func (b *memoryBackend) TimelineGet(before, after, channel string) (microsub.Tim
 	}, nil
 }
 
-//panic if s is not a slice
+// panic if s is not a slice
 func reverseSlice(s interface{}) {
 	size := reflect.ValueOf(s).Len()
 	swap := reflect.Swapper(s)
@@ -665,7 +524,7 @@ func (b *memoryBackend) Search(query string) ([]microsub.Feed, error) {
 		}
 		defer feedResp.Body.Close()
 
-		parsedFeed, err := b.feedHeader(fetchUrl.String(), feedResp.Header.Get("Content-Type"), feedResp.Body)
+		parsedFeed, err := fetch.FeedHeader(&fetch2{}, fetchUrl.String(), feedResp.Header.Get("Content-Type"), feedResp.Body)
 		if err != nil {
 			log.Printf("Error in parse of %s - %v\n", fetchUrl, err)
 			continue
@@ -684,9 +543,10 @@ func (b *memoryBackend) Search(query string) ([]microsub.Feed, error) {
 						log.Printf("Error in fetch of %s - %v\n", alt, err)
 						continue
 					}
+					// FIXME: don't defer in for loop (possible memory leak)
 					defer feedResp.Body.Close()
 
-					parsedFeed, err := b.feedHeader(alt, feedResp.Header.Get("Content-Type"), feedResp.Body)
+					parsedFeed, err := fetch.FeedHeader(&fetch2{}, alt, feedResp.Header.Get("Content-Type"), feedResp.Body)
 					if err != nil {
 						log.Printf("Error in parse of %s - %v\n", alt, err)
 						continue
@@ -706,7 +566,7 @@ func (b *memoryBackend) PreviewURL(previewURL string) (microsub.Timeline, error)
 	if err != nil {
 		return microsub.Timeline{}, fmt.Errorf("error while fetching %s: %v", previewURL, err)
 	}
-	items, err := b.feedItems(previewURL, resp.Header.Get("content-type"), resp.Body)
+	items, err := fetch.FeedItems(&fetch2{}, previewURL, resp.Header.Get("content-type"), resp.Body)
 	if err != nil {
 		return microsub.Timeline{}, fmt.Errorf("error while fetching %s: %v", previewURL, err)
 	}
@@ -748,3 +608,190 @@ func (b *memoryBackend) MarkRead(channel string, uids []string) error {
 
 	return nil
 }
+
+func (b *memoryBackend) ProcessContent(channel, fetchURL, contentType string, body io.Reader) error {
+	conn := pool.Get()
+	defer conn.Close()
+
+	items, err := fetch.FeedItems(&fetch2{}, fetchURL, contentType, body)
+	if err != nil {
+		return err
+	}
+
+	for _, item := range items {
+		item.Read = false
+		err = b.channelAddItemWithMatcher(conn, channel, item)
+		if err != nil {
+			log.Printf("ERROR: %s\n", err)
+		}
+	}
+
+	err = b.updateChannelUnreadCount(conn, channel)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Fetch3 fills stuff
+func (b *memoryBackend) Fetch3(channel, fetchURL string) (*http.Response, error) {
+	log.Printf("Fetching channel=%s fetchURL=%s\n", channel, fetchURL)
+	return Fetch2(fetchURL)
+}
+
+func (b *memoryBackend) channelAddItemWithMatcher(conn redis.Conn, channel string, item microsub.Item) error {
+	for channelKey, setting := range b.Settings {
+		if setting.IncludeRegex != "" {
+			included := false
+			includeRegex, err := regexp.Compile(setting.IncludeRegex)
+			if err != nil {
+				log.Printf("error in regexp: %q\n", includeRegex)
+			} else {
+				if item.Content != nil && includeRegex.MatchString(item.Content.Text) {
+					log.Printf("Included %#v\n", item)
+					included = true
+				}
+
+				if includeRegex.MatchString(item.Name) {
+					log.Printf("Included %#v\n", item)
+					included = true
+				}
+			}
+
+			if included {
+				b.channelAddItem(conn, channelKey, item)
+			}
+		}
+	}
+
+	if setting, e := b.Settings[channel]; e {
+		if setting.ExcludeRegex != "" {
+			excludeRegex, err := regexp.Compile(setting.ExcludeRegex)
+			if err != nil {
+				log.Printf("error in regexp: %q\n", excludeRegex)
+			} else {
+				if item.Content != nil && excludeRegex.MatchString(item.Content.Text) {
+					log.Printf("Excluded %#v\n", item)
+					return nil
+				}
+
+				if excludeRegex.MatchString(item.Name) {
+					log.Printf("Excluded %#v\n", item)
+					return nil
+				}
+			}
+		}
+	}
+
+	return b.channelAddItem(conn, channel, item)
+}
+
+func (b *memoryBackend) channelAddItem(conn redis.Conn, channel string, item microsub.Item) error {
+	zchannelKey := fmt.Sprintf("zchannel:%s:posts", channel)
+
+	if item.Published == "" {
+		item.Published = time.Now().Format(time.RFC3339)
+	}
+
+	data, err := json.Marshal(item)
+	if err != nil {
+		log.Printf("error while creating item for redis: %v\n", err)
+		return err
+	}
+
+	forRedis := redisItem{
+		ID:        item.ID,
+		Published: item.Published,
+		Read:      item.Read,
+		Data:      data,
+	}
+
+	itemKey := fmt.Sprintf("item:%s", item.ID)
+	_, err = redis.String(conn.Do("HMSET", redis.Args{}.Add(itemKey).AddFlat(&forRedis)...))
+	if err != nil {
+		return fmt.Errorf("error while writing item for redis: %v", err)
+	}
+
+	readChannelKey := fmt.Sprintf("channel:%s:read", channel)
+	isRead, err := redis.Bool(conn.Do("SISMEMBER", readChannelKey, itemKey))
+	if err != nil {
+		return err
+	}
+
+	if isRead {
+		return nil
+	}
+
+	score, err := time.Parse(time.RFC3339, item.Published)
+	if err != nil {
+		return fmt.Errorf("error can't parse %s as time", item.Published)
+	}
+
+	_, err = redis.Int64(conn.Do("ZADD", zchannelKey, score.Unix()*1.0, itemKey))
+	if err != nil {
+		return fmt.Errorf("error while zadding item %s to channel %s for redis: %v", itemKey, zchannelKey, err)
+	}
+
+	return nil
+}
+
+func (b *memoryBackend) updateChannelUnreadCount(conn redis.Conn, channel string) error {
+	if c, e := b.Channels[channel]; e {
+		zchannelKey := fmt.Sprintf("zchannel:%s:posts", channel)
+		unread, err := redis.Int(conn.Do("ZCARD", zchannelKey))
+		if err != nil {
+			return fmt.Errorf("error: while updating channel unread count for %s: %s", channel, err)
+		}
+		defer b.save()
+		c.Unread = unread
+		b.Channels[channel] = c
+	}
+	return nil
+}
+
+// Fetch2 fetches stuff
+func Fetch2(fetchURL string) (*http.Response, error) {
+	conn := pool.Get()
+	defer conn.Close()
+
+	if !strings.HasPrefix(fetchURL, "http") {
+		return nil, fmt.Errorf("error parsing %s as url, has no http(s) prefix", fetchURL)
+	}
+
+	u, err := url.Parse(fetchURL)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing %s as url: %s", fetchURL, err)
+	}
+
+	req, err := http.NewRequest("GET", u.String(), nil)
+
+	cacheKey := fmt.Sprintf("http_cache:%s", u.String())
+	data, err := redis.Bytes(conn.Do("GET", cacheKey))
+	if err == nil {
+		log.Printf("HIT %s\n", u.String())
+		rd := bufio.NewReader(bytes.NewReader(data))
+		return http.ReadResponse(rd, req)
+	}
+
+	log.Printf("MISS %s\n", u.String())
+
+	client := http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error while fetching %s: %s", u, err)
+	}
+	defer resp.Body.Close()
+
+	var b bytes.Buffer
+	resp.Write(&b)
+
+	cachedCopy := make([]byte, b.Len())
+	cur := b.Bytes()
+	copy(cachedCopy, cur)
+
+	conn.Do("SET", cacheKey, cachedCopy, "EX", 60*60)
+
+	cachedResp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(cachedCopy)), req)
+	return cachedResp, err
+}

cmd/eksterd/micropub.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 	"time"
 
+	"p83.nl/go/ekster/pkg/jf2"
 	"p83.nl/go/ekster/pkg/microsub"
 
 	"github.com/garyburd/redigo/redis"
@@ -91,7 +92,7 @@ func (h *micropubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 
-			item = mapToItem(simplifyMicroformat(&mfItem))
+			item = jf2.MapToItem(jf2.SimplifyMicroformat(&mfItem))
 			ok = true
 		} else if r.Header.Get("Content-Type") == "application/x-www-form-urlencoded" {
 			content := r.FormValue("content")

cmd/eksterd/null.go

@@ -28,10 +28,8 @@ type NullBackend struct {
 // ChannelsGetList gets no channels
 func (b *NullBackend) ChannelsGetList() ([]microsub.Channel, error) {
 	return []microsub.Channel{
-		microsub.Channel{UID: "0000", Name: "default", Unread: 0},
 		microsub.Channel{UID: "0001", Name: "notifications", Unread: 0},
-		microsub.Channel{UID: "1000", Name: "Friends", Unread: 0},
-		microsub.Channel{UID: "1001", Name: "Family", Unread: 0},
+		microsub.Channel{UID: "0000", Name: "default", Unread: 0},
 	}, nil
 }
 

cmd/eksterd/simplify.go

@@ -1,145 +0,0 @@
-/*
-   ekster - microsub server
-   Copyright (C) 2018  Peter Stuifzand
-
-   This program is free software: you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published by
-   the Free Software Foundation, either version 3 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-package main
-
-import (
-	"strings"
-
-	"willnorris.com/go/microformats"
-)
-
-func simplify(itemType string, item map[string][]interface{}) map[string]interface{} {
-	feedItem := make(map[string]interface{})
-
-	for k, v := range item {
-		if k == "bookmark-of" || k == "like-of" || k == "repost-of" || k == "in-reply-to" {
-			if value, ok := v[0].(*microformats.Microformat); ok {
-				feedItem[k] = value.Value
-			} else {
-				feedItem[k] = v
-			}
-		} else if k == "content" {
-			if content, ok := v[0].(map[string]interface{}); ok {
-				if text, e := content["value"]; e {
-					delete(content, "value")
-					content["text"] = text
-				}
-				feedItem[k] = content
-			}
-		} else if k == "photo" {
-			if itemType == "card" {
-				if len(v) >= 1 {
-					if value, ok := v[0].(string); ok {
-						feedItem[k] = value
-					}
-				}
-			} else {
-				feedItem[k] = v
-			}
-		} else if k == "video" {
-			feedItem[k] = v
-		} else if k == "featured" {
-			feedItem[k] = v
-		} else if k == "checkin" || k == "author" {
-			if value, ok := v[0].(*microformats.Microformat); ok {
-				card := make(map[string]string)
-				card["type"] = "card"
-				for ik, vk := range value.Properties {
-					if p, ok := vk[0].(string); ok {
-						card[ik] = p
-					}
-				}
-				feedItem[k] = card
-			}
-		} else if value, ok := v[0].(*microformats.Microformat); ok {
-			mType := value.Type[0][2:]
-			m := simplify(mType, value.Properties)
-			m["type"] = mType
-			feedItem[k] = m
-		} else if value, ok := v[0].(string); ok {
-			feedItem[k] = value
-		} else if value, ok := v[0].(map[string]interface{}); ok {
-			feedItem[k] = value
-		} else if value, ok := v[0].([]interface{}); ok {
-			feedItem[k] = value
-		}
-	}
-
-	// Remove "name" when it's equals to "content[text]"
-	if name, e := feedItem["name"]; e {
-		if content, e2 := feedItem["content"]; e2 {
-			if contentMap, ok := content.(map[string]interface{}); ok {
-				if text, e3 := contentMap["text"]; e3 {
-					if strings.TrimSpace(name.(string)) == strings.TrimSpace(text.(string)) {
-						delete(feedItem, "name")
-					}
-				}
-			}
-		}
-	}
-
-	return feedItem
-}
-
-func simplifyMicroformat(item *microformats.Microformat) map[string]interface{} {
-	itemType := item.Type[0][2:]
-	newItem := simplify(itemType, item.Properties)
-	newItem["type"] = itemType
-
-	children := []map[string]interface{}{}
-
-	if len(item.Children) > 0 {
-		for _, c := range item.Children {
-			child := simplifyMicroformat(c)
-			if c, e := child["children"]; e {
-				if ar, ok := c.([]map[string]interface{}); ok {
-					children = append(children, ar...)
-				}
-				delete(child, "children")
-			}
-			children = append(children, child)
-		}
-
-		newItem["children"] = children
-	}
-
-	return newItem
-}
-
-func simplifyMicroformatData(md *microformats.Data) []map[string]interface{} {
-	items := []map[string]interface{}{}
-	for _, item := range md.Items {
-		if len(item.Type) >= 1 && item.Type[0] == "h-feed" {
-			for _, childItem := range item.Children {
-				newItem := simplifyMicroformat(childItem)
-				items = append(items, newItem)
-			}
-			return items
-		}
-
-		newItem := simplifyMicroformat(item)
-		items = append(items, newItem)
-		if c, e := newItem["children"]; e {
-			if ar, ok := c.([]map[string]interface{}); ok {
-				items = append(items, ar...)
-			}
-			delete(newItem, "children")
-		}
-	}
-	return items
-}

cmd/jf2test/main.go

@@ -0,0 +1 @@
+package jf2test

pkg/fetch/fetch.go

@@ -16,32 +16,28 @@
    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
-package main
+package fetch
 
 import (
-	"bufio"
-	"bytes"
 	"encoding/hex"
 	"encoding/json"
-	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
-	"net/http"
 	"net/url"
-	"regexp"
-	"rss"
 	"strings"
 	"time"
 
+	"rss"
+
+	"p83.nl/go/ekster/pkg/jf2"
+	"p83.nl/go/ekster/pkg/jsonfeed"
 	"p83.nl/go/ekster/pkg/microsub"
 
-	"github.com/garyburd/redigo/redis"
-	"p83.nl/go/ekster/pkg/jsonfeed"
 	"willnorris.com/go/microformats"
 )
 
-func (b *memoryBackend) feedHeader(fetchURL, contentType string, body io.Reader) (microsub.Feed, error) {
+func FeedHeader(fetcher Fetcher, fetchURL, contentType string, body io.Reader) (microsub.Feed, error) {
 	log.Printf("ProcessContent %s\n", fetchURL)
 	log.Println("Found " + contentType)
 
@@ -53,7 +49,7 @@ func (b *memoryBackend) feedHeader(fetchURL, contentType string, body io.Reader)
 
 	if strings.HasPrefix(contentType, "text/html") {
 		data := microformats.Parse(body, u)
-		results := simplifyMicroformatData(data)
+		results := jf2.SimplifyMicroformatData(data)
 		found := -1
 		for i, r := range results {
 			if r["type"] == "card" {
@@ -67,7 +63,7 @@ func (b *memoryBackend) feedHeader(fetchURL, contentType string, body io.Reader)
 
 			if as, ok := card.(string); ok {
 				if strings.HasPrefix(as, "http") {
-					resp, err := Fetch2(fetchURL)
+					resp, err := fetcher.Fetch(fetchURL)
 					if err != nil {
 						return feed, err
 					}
@@ -75,7 +71,7 @@ func (b *memoryBackend) feedHeader(fetchURL, contentType string, body io.Reader)
 					u, _ := url.Parse(fetchURL)
 
 					md := microformats.Parse(resp.Body, u)
-					author := simplifyMicroformatData(md)
+					author := jf2.SimplifyMicroformatData(md)
 					for _, a := range author {
 						if a["type"] == "card" {
 							card = a
@@ -154,7 +150,7 @@ func (b *memoryBackend) feedHeader(fetchURL, contentType string, body io.Reader)
 	return feed, nil
 }
 
-func (b *memoryBackend) feedItems(fetchURL, contentType string, body io.Reader) ([]microsub.Item, error) {
+func FeedItems(fetcher Fetcher, fetchURL, contentType string, body io.Reader) ([]microsub.Item, error) {
 	log.Printf("ProcessContent %s\n", fetchURL)
 	log.Println("Found " + contentType)
 
@@ -164,7 +160,7 @@ func (b *memoryBackend) feedItems(fetchURL, contentType string, body io.Reader)
 
 	if strings.HasPrefix(contentType, "text/html") {
 		data := microformats.Parse(body, u)
-		results := simplifyMicroformatData(data)
+		results := jf2.SimplifyMicroformatData(data)
 		found := -1
 		for {
 			for i, r := range results {
@@ -190,7 +186,7 @@ func (b *memoryBackend) feedItems(fetchURL, contentType string, body io.Reader)
 		for i, r := range results {
 			if as, ok := r["author"].(string); ok {
 				if r["type"] == "entry" && strings.HasPrefix(as, "http") {
-					resp, err := Fetch2(fetchURL)
+					resp, err := fetcher.Fetch(fetchURL)
 					if err != nil {
 						return items, err
 					}
@@ -198,7 +194,7 @@ func (b *memoryBackend) feedItems(fetchURL, contentType string, body io.Reader)
 					u, _ := url.Parse(fetchURL)
 
 					md := microformats.Parse(resp.Body, u)
-					author := simplifyMicroformatData(md)
+					author := jf2.SimplifyMicroformatData(md)
 					for _, a := range author {
 						if a["type"] == "card" {
 							results[i]["author"] = a
@@ -217,11 +213,11 @@ func (b *memoryBackend) feedItems(fetchURL, contentType string, body io.Reader)
 				r["_id"] = hex.EncodeToString([]byte(uid.(string)))
 			} else {
 				continue
-				//r["_id"] = "" // generate random value
+				// r["_id"] = "" // generate random value
 			}
 
 			// mapToItem adds published
-			item := mapToItem(r)
+			item := jf2.MapToItem(r)
 			items = append(items, item)
 		}
 	} else if strings.HasPrefix(contentType, "application/json") { // json feed?
@@ -332,197 +328,3 @@ func (b *memoryBackend) feedItems(fetchURL, contentType string, body io.Reader)
 
 	return items, nil
 }
-
-func (b *memoryBackend) ProcessContent(channel, fetchURL, contentType string, body io.Reader) error {
-	conn := pool.Get()
-	defer conn.Close()
-
-	items, err := b.feedItems(fetchURL, contentType, body)
-	if err != nil {
-		return err
-	}
-
-	for _, item := range items {
-		item.Read = false
-		err = b.channelAddItemWithMatcher(conn, channel, item)
-		if err != nil {
-			log.Printf("ERROR: %s\n", err)
-		}
-	}
-
-	err = b.updateChannelUnreadCount(conn, channel)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// Fetch3 fills stuff
-func (b *memoryBackend) Fetch3(channel, fetchURL string) (*http.Response, error) {
-	log.Printf("Fetching channel=%s fetchURL=%s\n", channel, fetchURL)
-	return Fetch2(fetchURL)
-}
-
-func (b *memoryBackend) channelAddItemWithMatcher(conn redis.Conn, channel string, item microsub.Item) error {
-	for channelKey, setting := range b.Settings {
-		if setting.IncludeRegex != "" {
-			included := false
-			includeRegex, err := regexp.Compile(setting.IncludeRegex)
-			if err != nil {
-				log.Printf("error in regexp: %q\n", includeRegex)
-			} else {
-				if item.Content != nil && includeRegex.MatchString(item.Content.Text) {
-					log.Printf("Included %#v\n", item)
-					included = true
-				}
-
-				if includeRegex.MatchString(item.Name) {
-					log.Printf("Included %#v\n", item)
-					included = true
-				}
-			}
-
-			if included {
-				b.channelAddItem(conn, channelKey, item)
-			}
-		}
-	}
-
-	if setting, e := b.Settings[channel]; e {
-		if setting.ExcludeRegex != "" {
-			excludeRegex, err := regexp.Compile(setting.ExcludeRegex)
-			if err != nil {
-				log.Printf("error in regexp: %q\n", excludeRegex)
-			} else {
-				if item.Content != nil && excludeRegex.MatchString(item.Content.Text) {
-					log.Printf("Excluded %#v\n", item)
-					return nil
-				}
-
-				if excludeRegex.MatchString(item.Name) {
-					log.Printf("Excluded %#v\n", item)
-					return nil
-				}
-			}
-		}
-	}
-
-	return b.channelAddItem(conn, channel, item)
-}
-
-func (b *memoryBackend) channelAddItem(conn redis.Conn, channel string, item microsub.Item) error {
-	zchannelKey := fmt.Sprintf("zchannel:%s:posts", channel)
-
-	if item.Published == "" {
-		item.Published = time.Now().Format(time.RFC3339)
-	}
-
-	data, err := json.Marshal(item)
-	if err != nil {
-		log.Printf("error while creating item for redis: %v\n", err)
-		return err
-	}
-
-	forRedis := redisItem{
-		ID:        item.ID,
-		Published: item.Published,
-		Read:      item.Read,
-		Data:      data,
-	}
-
-	itemKey := fmt.Sprintf("item:%s", item.ID)
-	_, err = redis.String(conn.Do("HMSET", redis.Args{}.Add(itemKey).AddFlat(&forRedis)...))
-	if err != nil {
-		return fmt.Errorf("error while writing item for redis: %v", err)
-	}
-
-	readChannelKey := fmt.Sprintf("channel:%s:read", channel)
-	isRead, err := redis.Bool(conn.Do("SISMEMBER", readChannelKey, itemKey))
-	if err != nil {
-		return err
-	}
-
-	if isRead {
-		return nil
-	}
-
-	score, err := time.Parse(time.RFC3339, item.Published)
-	if err != nil {
-		return fmt.Errorf("error can't parse %s as time", item.Published)
-	}
-
-	_, err = redis.Int64(conn.Do("ZADD", zchannelKey, score.Unix()*1.0, itemKey))
-	if err != nil {
-		return fmt.Errorf("error while zadding item %s to channel %s for redis: %v", itemKey, zchannelKey, err)
-	}
-
-	return nil
-}
-
-func (b *memoryBackend) updateChannelUnreadCount(conn redis.Conn, channel string) error {
-	if c, e := b.Channels[channel]; e {
-		zchannelKey := fmt.Sprintf("zchannel:%s:posts", channel)
-		unread, err := redis.Int(conn.Do("ZCARD", zchannelKey))
-		if err != nil {
-			return fmt.Errorf("error: while updating channel unread count for %s: %s", channel, err)
-		}
-		defer b.save()
-		c.Unread = unread
-		b.Channels[channel] = c
-	}
-	return nil
-}
-
-type redisItem struct {
-	ID        string
-	Published string
-	Read      bool
-	Data      []byte
-}
-
-// Fetch2 fetches stuff
-func Fetch2(fetchURL string) (*http.Response, error) {
-	conn := pool.Get()
-	defer conn.Close()
-
-	if !strings.HasPrefix(fetchURL, "http") {
-		return nil, fmt.Errorf("error parsing %s as url, has no http(s) prefix", fetchURL)
-	}
-
-	u, err := url.Parse(fetchURL)
-	if err != nil {
-		return nil, fmt.Errorf("error parsing %s as url: %s", fetchURL, err)
-	}
-
-	req, err := http.NewRequest("GET", u.String(), nil)
-
-	cacheKey := fmt.Sprintf("http_cache:%s", u.String())
-	data, err := redis.Bytes(conn.Do("GET", cacheKey))
-	if err == nil {
-		log.Printf("HIT %s\n", u.String())
-		rd := bufio.NewReader(bytes.NewReader(data))
-		return http.ReadResponse(rd, req)
-	}
-
-	log.Printf("MISS %s\n", u.String())
-
-	client := http.Client{}
-	resp, err := client.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("error while fetching %s: %s", u, err)
-	}
-	defer resp.Body.Close()
-
-	var b bytes.Buffer
-	resp.Write(&b)
-
-	cachedCopy := make([]byte, b.Len())
-	cur := b.Bytes()
-	copy(cachedCopy, cur)
-
-	conn.Do("SET", cacheKey, cachedCopy, "EX", 60*60)
-
-	cachedResp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(cachedCopy)), req)
-	return cachedResp, err
-}

pkg/fetch/fetcher.go

@@ -0,0 +1,7 @@
+package fetch
+
+import "net/http"
+
+type Fetcher interface {
+	Fetch(url string) (*http.Response, error)
+}

pkg/jf2/simplify.go

@@ -0,0 +1,318 @@
+/*
+   ekster - microsub server
+   Copyright (C) 2018  Peter Stuifzand
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package jf2
+
+import (
+	"fmt"
+	"log"
+	"strings"
+	"time"
+
+	"p83.nl/go/ekster/pkg/microsub"
+
+	"willnorris.com/go/microformats"
+)
+
+func simplify(itemType string, item map[string][]interface{}) map[string]interface{} {
+	feedItem := make(map[string]interface{})
+
+	for k, v := range item {
+		if k == "bookmark-of" || k == "like-of" || k == "repost-of" || k == "in-reply-to" {
+			if value, ok := v[0].(*microformats.Microformat); ok {
+				feedItem[k] = value.Value
+			} else {
+				feedItem[k] = v
+			}
+		} else if k == "content" {
+			if content, ok := v[0].(map[string]interface{}); ok {
+				if text, e := content["value"]; e {
+					delete(content, "value")
+					content["text"] = text
+				}
+				feedItem[k] = content
+			}
+		} else if k == "photo" {
+			if itemType == "card" {
+				if len(v) >= 1 {
+					if value, ok := v[0].(string); ok {
+						feedItem[k] = value
+					}
+				}
+			} else {
+				feedItem[k] = v
+			}
+		} else if k == "video" {
+			feedItem[k] = v
+		} else if k == "featured" {
+			feedItem[k] = v
+		} else if k == "checkin" || k == "author" {
+			card, err := simplifyCard(v)
+			if err != nil {
+				log.Println(err)
+				continue
+			}
+
+			feedItem[k] = card
+		} else if value, ok := v[0].(*microformats.Microformat); ok {
+			mType := value.Type[0][2:]
+			m := simplify(mType, value.Properties)
+			m["type"] = mType
+			feedItem[k] = m
+		} else if value, ok := v[0].(string); ok {
+			feedItem[k] = value
+		} else if value, ok := v[0].(map[string]interface{}); ok {
+			feedItem[k] = value
+		} else if value, ok := v[0].([]interface{}); ok {
+			feedItem[k] = value
+		}
+	}
+
+	// Remove "name" when it's equals to "content[text]"
+	if name, e := feedItem["name"]; e {
+		if content, e2 := feedItem["content"]; e2 {
+			if contentMap, ok := content.(map[string]interface{}); ok {
+				if text, e3 := contentMap["text"]; e3 {
+					if strings.TrimSpace(name.(string)) == strings.TrimSpace(text.(string)) {
+						delete(feedItem, "name")
+					}
+				}
+			}
+		}
+	}
+
+	return feedItem
+}
+func simplifyCard(v []interface{}) (map[string]string, error) {
+	if value, ok := v[0].(*microformats.Microformat); ok {
+		card := make(map[string]string)
+		card["type"] = "card"
+		for ik, vk := range value.Properties {
+			if p, ok := vk[0].(string); ok {
+				card[ik] = p
+			}
+		}
+		return card, nil
+	}
+	return nil, fmt.Errorf("not convertable to a card %q", v)
+}
+
+func SimplifyMicroformat(item *microformats.Microformat) map[string]interface{} {
+	itemType := item.Type[0][2:]
+	newItem := simplify(itemType, item.Properties)
+	newItem["type"] = itemType
+
+	children := []map[string]interface{}{}
+
+	if len(item.Children) > 0 {
+		for _, c := range item.Children {
+			child := SimplifyMicroformat(c)
+			if c, e := child["children"]; e {
+				if ar, ok := c.([]map[string]interface{}); ok {
+					children = append(children, ar...)
+				}
+				delete(child, "children")
+			}
+			children = append(children, child)
+		}
+
+		newItem["children"] = children
+	}
+
+	return newItem
+}
+
+func SimplifyMicroformatData(md *microformats.Data) []map[string]interface{} {
+	items := []map[string]interface{}{}
+	for _, item := range md.Items {
+		if len(item.Type) >= 1 && item.Type[0] == "h-feed" {
+			for _, childItem := range item.Children {
+				newItem := SimplifyMicroformat(childItem)
+				items = append(items, newItem)
+			}
+			return items
+		}
+
+		newItem := SimplifyMicroformat(item)
+		items = append(items, newItem)
+		if c, e := newItem["children"]; e {
+			if ar, ok := c.([]map[string]interface{}); ok {
+				items = append(items, ar...)
+			}
+			delete(newItem, "children")
+		}
+	}
+	return items
+}
+
+func MapToAuthor(result map[string]string) *microsub.Card {
+	item := &microsub.Card{}
+	item.Type = "card"
+	if name, e := result["name"]; e {
+		item.Name = name
+	}
+	if u, e := result["url"]; e {
+		item.URL = u
+	}
+	if photo, e := result["photo"]; e {
+		item.Photo = photo
+	}
+	if value, e := result["longitude"]; e {
+		item.Longitude = value
+	}
+	if value, e := result["latitude"]; e {
+		item.Latitude = value
+	}
+	if value, e := result["country-name"]; e {
+		item.CountryName = value
+	}
+	if value, e := result["locality"]; e {
+		item.Locality = value
+	}
+	return item
+}
+
+func MapToItem(result map[string]interface{}) microsub.Item {
+	item := microsub.Item{}
+
+	item.Type = "entry"
+
+	if name, e := result["name"]; e {
+		item.Name = name.(string)
+	}
+
+	if url, e := result["url"]; e {
+		item.URL = url.(string)
+	}
+
+	if uid, e := result["uid"]; e {
+		item.UID = uid.(string)
+	}
+
+	if author, e := result["author"]; e {
+		item.Author = MapToAuthor(author.(map[string]string))
+	}
+
+	if checkin, e := result["checkin"]; e {
+		item.Checkin = MapToAuthor(checkin.(map[string]string))
+	}
+
+	if content, e := result["content"]; e {
+		itemContent := &microsub.Content{}
+		set := false
+		if c, ok := content.(map[string]interface{}); ok {
+			if html, e2 := c["html"]; e2 {
+				itemContent.HTML = html.(string)
+				set = true
+			}
+			if text, e2 := c["value"]; e2 {
+				itemContent.Text = text.(string)
+				set = true
+			}
+		}
+		if set {
+			item.Content = itemContent
+		}
+	}
+
+	// TODO: Check how to improve this
+
+	if value, e := result["like-of"]; e {
+		for _, v := range value.([]interface{}) {
+			if u, ok := v.(string); ok {
+				item.LikeOf = append(item.LikeOf, u)
+			}
+		}
+	}
+
+	if value, e := result["repost-of"]; e {
+		if repost, ok := value.(string); ok {
+			item.RepostOf = append(item.RepostOf, repost)
+		} else if repost, ok := value.([]interface{}); ok {
+			for _, v := range repost {
+				if u, ok := v.(string); ok {
+					item.RepostOf = append(item.RepostOf, u)
+				}
+			}
+		}
+	}
+
+	if value, e := result["bookmark-of"]; e {
+		for _, v := range value.([]interface{}) {
+			if u, ok := v.(string); ok {
+				item.BookmarkOf = append(item.BookmarkOf, u)
+			}
+		}
+	}
+
+	if value, e := result["in-reply-to"]; e {
+		if replyTo, ok := value.(string); ok {
+			item.InReplyTo = append(item.InReplyTo, replyTo)
+		} else if valueArray, ok := value.([]interface{}); ok {
+			for _, v := range valueArray {
+				if replyTo, ok := v.(string); ok {
+					item.InReplyTo = append(item.InReplyTo, replyTo)
+				} else if cite, ok := v.(map[string]interface{}); ok {
+					item.InReplyTo = append(item.InReplyTo, cite["url"].(string))
+				}
+			}
+		}
+	}
+
+	if value, e := result["photo"]; e {
+		for _, v := range value.([]interface{}) {
+			item.Photo = append(item.Photo, v.(string))
+		}
+	}
+
+	if value, e := result["category"]; e {
+		if cats, ok := value.([]string); ok {
+			for _, v := range cats {
+				item.Category = append(item.Category, v)
+			}
+		} else if cats, ok := value.([]interface{}); ok {
+			for _, v := range cats {
+				if cat, ok := v.(string); ok {
+					item.Category = append(item.Category, cat)
+				} else if cat, ok := v.(map[string]interface{}); ok {
+					item.Category = append(item.Category, cat["value"].(string))
+				}
+			}
+		} else if cat, ok := value.(string); ok {
+			item.Category = append(item.Category, cat)
+		}
+	}
+
+	if published, e := result["published"]; e {
+		item.Published = published.(string)
+	} else {
+		item.Published = time.Now().Format(time.RFC3339)
+	}
+
+	if updated, e := result["updated"]; e {
+		item.Updated = updated.(string)
+	}
+
+	if id, e := result["_id"]; e {
+		item.ID = id.(string)
+	}
+	if read, e := result["_is_read"]; e {
+		item.Read = read.(bool)
+	}
+
+	return item
+}

pkg/jf2/simplify_test.go

@@ -15,7 +15,7 @@
    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
-package main
+package jf2
 
 import (
 	"log"
@@ -40,7 +40,7 @@ func TestInReplyTo(t *testing.T) {
 	}
 
 	data := microformats.Parse(f, u)
-	results := simplifyMicroformatData(data)
+	results := SimplifyMicroformatData(data)
 
 	if results[0]["type"] != "entry" {
 		t.Fatalf("not an h-entry, but %s", results[0]["type"])