Implement updating content from websub hub
This commit is contained in:
parent
fc179be0b5
commit
42a73d6ccd
|
|
@ -22,6 +22,7 @@ import (
|
|||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
|
|
@ -47,21 +48,12 @@ func init() {
|
|||
cache = make(map[string]cacheItem)
|
||||
}
|
||||
|
||||
// Fetch3 fills stuff
|
||||
func (b *memoryBackend) Fetch3(channel, fetchURL string) error {
|
||||
log.Printf("Fetching channel=%s fetchURL=%s\n", channel, fetchURL)
|
||||
|
||||
resp, err := Fetch2(fetchURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
func (b *memoryBackend) ProcessContent(channel, fetchURL, contentType string, body io.Reader) error {
|
||||
u, _ := url.Parse(fetchURL)
|
||||
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
log.Println("Found " + contentType)
|
||||
if strings.HasPrefix(contentType, "text/html") {
|
||||
data := microformats.Parse(resp.Body, u)
|
||||
data := microformats.Parse(body, u)
|
||||
results := simplifyMicroformatData(data)
|
||||
found := -1
|
||||
for {
|
||||
|
|
@ -128,8 +120,8 @@ func (b *memoryBackend) Fetch3(channel, fetchURL string) error {
|
|||
}
|
||||
} else if strings.HasPrefix(contentType, "application/json") { // json feed?
|
||||
var feed JSONFeed
|
||||
dec := json.NewDecoder(resp.Body)
|
||||
err = dec.Decode(&feed)
|
||||
dec := json.NewDecoder(body)
|
||||
err := dec.Decode(&feed)
|
||||
if err != nil {
|
||||
log.Printf("Error while parsing json feed: %s\n", err)
|
||||
return err
|
||||
|
|
@ -146,7 +138,7 @@ func (b *memoryBackend) Fetch3(channel, fetchURL string) error {
|
|||
b.channelAddItem(channel, item)
|
||||
}
|
||||
} else if strings.HasPrefix(contentType, "text/xml") || strings.HasPrefix(contentType, "application/rss+xml") || strings.HasPrefix(contentType, "application/atom+xml") {
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
body, err := ioutil.ReadAll(body)
|
||||
if err != nil {
|
||||
log.Printf("Error while parsing rss/atom feed: %s\n", err)
|
||||
return err
|
||||
|
|
@ -171,10 +163,22 @@ func (b *memoryBackend) Fetch3(channel, fetchURL string) error {
|
|||
} else {
|
||||
log.Printf("Unknown Content-Type: %s\n", contentType)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Fetch3 fills stuff
|
||||
func (b *memoryBackend) Fetch3(channel, fetchURL string) error {
|
||||
log.Printf("Fetching channel=%s fetchURL=%s\n", channel, fetchURL)
|
||||
|
||||
resp, err := Fetch2(fetchURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
return b.ProcessContent(channel, fetchURL, resp.Header.Get("Content-Type"), resp.Body)
|
||||
}
|
||||
|
||||
func (b *memoryBackend) channelAddItem(channel string, item microsub.Item) {
|
||||
log.Printf("Adding item to channel %s\n", channel)
|
||||
log.Println(item)
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/hmac"
|
||||
"crypto/sha1"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
|
|
@ -14,9 +16,9 @@ import (
|
|||
|
||||
// HubBackend handles information for the incoming handler
|
||||
type HubBackend interface {
|
||||
CreateFeed(url string, contentType string) (int64, error)
|
||||
CreateFeed(url, channel string) (int64, error)
|
||||
GetSecret(id int64) string
|
||||
UpdateFeed(feedID int64, contentType string, content []byte) error
|
||||
UpdateFeed(feedID int64, contentType string, body io.Reader) error
|
||||
}
|
||||
|
||||
type incomingHandler struct {
|
||||
|
|
@ -92,17 +94,9 @@ func (h *incomingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
|
||||
ct := r.Header.Get("Content-Type")
|
||||
if strings.HasPrefix(ct, "application/rss+xml") {
|
||||
// RSS parsing
|
||||
h.Backend.UpdateFeed(feed, ct, feedContent)
|
||||
} else if strings.HasPrefix(ct, "application/atom+xml") {
|
||||
// Atom parsing
|
||||
h.Backend.UpdateFeed(feed, ct, feedContent)
|
||||
} else if strings.HasPrefix(ct, "text/html") {
|
||||
// h-entry parsing
|
||||
h.Backend.UpdateFeed(feed, ct, feedContent)
|
||||
} else {
|
||||
http.Error(w, fmt.Sprintf("Unknown format of body: %s", ct), 400)
|
||||
err = h.Backend.UpdateFeed(feed, ct, bytes.NewBuffer(feedContent))
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("Unknown format of body: %s (%s)", ct, err), 400)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import (
|
|||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
|
|
@ -29,8 +30,6 @@ import (
|
|||
"regexp"
|
||||
"time"
|
||||
|
||||
"rss"
|
||||
|
||||
"github.com/garyburd/redigo/redis"
|
||||
"github.com/pstuifzand/microsub-server/microsub"
|
||||
)
|
||||
|
|
@ -53,7 +52,8 @@ type microsubHandler struct {
|
|||
}
|
||||
|
||||
type hubIncomingBackend struct {
|
||||
conn redis.Conn
|
||||
backend *memoryBackend
|
||||
conn redis.Conn
|
||||
}
|
||||
|
||||
const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||
|
|
@ -76,7 +76,7 @@ func (h *hubIncomingBackend) GetSecret(id int64) string {
|
|||
|
||||
var hubURL = "https://hub.stuifzandapp.com/"
|
||||
|
||||
func (h *hubIncomingBackend) CreateFeed(topic string, contentType string) (int64, error) {
|
||||
func (h *hubIncomingBackend) CreateFeed(topic string, channel string) (int64, error) {
|
||||
id, err := redis.Int64(h.conn.Do("INCR", "feed:next_id"))
|
||||
|
||||
if err != nil {
|
||||
|
|
@ -84,7 +84,7 @@ func (h *hubIncomingBackend) CreateFeed(topic string, contentType string) (int64
|
|||
}
|
||||
|
||||
h.conn.Do("HSET", fmt.Sprintf("feed:%d", id), "url", topic)
|
||||
h.conn.Do("HSET", fmt.Sprintf("feed:%d", id), "type", contentType)
|
||||
h.conn.Do("HSET", fmt.Sprintf("feed:%d", id), "channel", channel)
|
||||
secret := randStringBytes(16)
|
||||
h.conn.Do("HSET", fmt.Sprintf("feed:%d", id), "secret", secret)
|
||||
|
||||
|
|
@ -105,81 +105,20 @@ func (h *hubIncomingBackend) CreateFeed(topic string, contentType string) (int64
|
|||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
filename := fmt.Sprintf("backend/feeds/%d.json", id)
|
||||
|
||||
feed, err := rss.Fetch(topic)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
os.MkdirAll("backend/feeds", 0755)
|
||||
|
||||
f, err := os.Create(filename)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
defer f.Close()
|
||||
|
||||
out := json.NewEncoder(f)
|
||||
err = out.Encode(feed)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func readFeedFile(filename string) (*rss.Feed, error) {
|
||||
f, err := os.Open(filename)
|
||||
func (h *hubIncomingBackend) UpdateFeed(feedID int64, contentType string, body io.Reader) error {
|
||||
u, err := redis.String(h.conn.Do("HGET", fmt.Sprintf("feed:%d", feedID), "url"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
var feed *rss.Feed
|
||||
out := json.NewDecoder(f)
|
||||
err = out.Decode(feed)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return feed, err
|
||||
}
|
||||
|
||||
func writeFeedFile(filename string, feed *rss.Feed) error {
|
||||
f, err := os.Create(filename)
|
||||
channel, err := redis.String(h.conn.Do("HGET", fmt.Sprintf("feed:%d", feedID), "channel"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer f.Close()
|
||||
|
||||
out := json.NewEncoder(f)
|
||||
err = out.Encode(feed)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *hubIncomingBackend) UpdateFeed(feedID int64, contentType string, content []byte) error {
|
||||
_, err := redis.String(h.conn.Do("HGET", fmt.Sprintf("feed:%d", feedID), "url"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
filename := fmt.Sprintf("backend/feeds/%d.json", feedID)
|
||||
|
||||
feed, err := readFeedFile(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = feed.UpdateWithContent(content)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = writeFeedFile(filename, feed)
|
||||
h.backend.ProcessContent(channel, u, contentType, body)
|
||||
|
||||
return err
|
||||
}
|
||||
|
|
@ -272,7 +211,7 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||
} else if action == "follow" {
|
||||
uid := values.Get("channel")
|
||||
url := values.Get("url")
|
||||
h.HubIncomingBackend.CreateFeed(url, "application/rss+xml")
|
||||
h.HubIncomingBackend.CreateFeed(url, uid)
|
||||
feed := h.Backend.FollowURL(uid, url)
|
||||
w.Header().Add("Content-Type", "application/json")
|
||||
jw := json.NewEncoder(w)
|
||||
|
|
@ -360,7 +299,7 @@ func main() {
|
|||
|
||||
backend = loadMemoryBackend(conn)
|
||||
|
||||
hubBackend := hubIncomingBackend{conn}
|
||||
hubBackend := hubIncomingBackend{backend.(*memoryBackend), conn}
|
||||
|
||||
http.Handle("/microsub", µsubHandler{
|
||||
Backend: backend,
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user