Add caching for Fetch2

cmd/server/fetch.go

@@ -19,6 +19,8 @@
 package main
 
 import (
+	"bufio"
+	"bytes"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
@@ -38,7 +40,7 @@ import (
 )
 
 type cacheItem struct {
-	item    *microformats.Data
+	item    []byte
 	created time.Time
 }
 
@@ -260,12 +262,37 @@ func Fetch2(fetchURL string) (*http.Response, error) {
 		return nil, fmt.Errorf("error parsing %s as url: %s", fetchURL, err)
 	}
 
-	resp, err := http.Get(u.String())
+	req, err := http.NewRequest("GET", u.String(), nil)
+
+	if data, e := cache[u.String()]; e {
+		if data.created.After(time.Now().Add(time.Minute * -10)) {
+			log.Printf("HIT %s - %s\n", u.String(), time.Now().Sub(data.created).String())
+			rd := bufio.NewReader(bytes.NewReader(data.item))
+			return http.ReadResponse(rd, req)
+		} else {
+			log.Printf("EXPIRE %s\n", u.String())
+			delete(cache, u.String())
+		}
+	} else {
+		log.Printf("MISS %s\n", u.String())
+	}
+
+	client := http.Client{}
+	resp, err := client.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("error while fetching %s: %s", u, err)
 	}
 
-	return resp, err
+	var b bytes.Buffer
+	resp.Write(&b)
+
+	cachedCopy := make([]byte, b.Len())
+	cur := b.Bytes()
+	copy(cachedCopy, cur)
+	cache[u.String()] = cacheItem{item: cachedCopy, created: time.Now()}
+
+	cachedResp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(cachedCopy)), req)
+	return cachedResp, err
 }
 
 func Fetch(fetchURL string) []microsub.Item {

cmd/server/memory.go

@@ -417,6 +417,7 @@ func (b *memoryBackend) Search(query string) []microsub.Feed {
 	feeds := []microsub.Feed{}
 
 	for _, u := range urls {
+		log.Println(u)
 		resp, err := Fetch2(u)
 		if err != nil {
 			log.Printf("Error while fetching %s: %v\n", u, err)

cmd/server/null.go

@@ -68,7 +68,7 @@ func (b *NullBackend) FollowGetList(uid string) []microsub.Feed {
 }
 
 func (b *NullBackend) FollowURL(uid string, url string) microsub.Feed {
-	return microsub.Feed{"feed", url}
+	return microsub.Feed{Type: "feed", URL: url}
 }
 
 func (b *NullBackend) UnfollowURL(uid string, url string) {

microsub/protocol.go

@@ -38,8 +38,8 @@ type Channel struct {
 }
 
 type Author struct {
-	Filled bool   `json:"-"`
-	Type   string `json:"type"`
+	Filled bool   `json:"-,omitempty"`
+	Type   string `json:"type,omitempty"`
 	Name   string `json:"name,omitempty"`
 	URL    string `json:"url,omitempty"`
 	Photo  string `json:"photo,omitempty"`
@@ -86,8 +86,12 @@ type Timeline struct {
 }
 
 type Feed struct {
-	Type string `json:"type"`
-	URL  string `json:"url"`
+	Type        string `json:"type"`
+	URL         string `json:"url"`
+	Name        string `json:"name,omitempty"`
+	Photo       string `json:"photo,omitempty"`
+	Description string `json:"description,omitempty"`
+	Author      Author `json:"author,omitempty"`
 }
 
 // Microsub is the main protocol that should be implemented by a backend