diff --git a/cmd/server/auth.go b/cmd/server/auth.go new file mode 100644 index 0000000..6392c34 --- /dev/null +++ b/cmd/server/auth.go @@ -0,0 +1,106 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "regexp" + "time" + + "github.com/garyburd/redigo/redis" +) + +// TokenResponse is the information that we get back from the token endpoint of the user... +type TokenResponse struct { + Me string `json:"me"` + ClientID string `json:"client_id"` + Scope string `json:"scope"` + IssuedAt int64 `json:"issued_at"` + Nonce int64 `json:"nonce"` +} + +var authHeaderRegex = regexp.MustCompile("^Bearer (.+)$") + +func (h *microsubHandler) cachedCheckAuthToken(header string, r *TokenResponse) bool { + r.Me = "https://publog.stuifzandapp.com/" + log.Println("Cached checking Auth Token") + + tokens := authHeaderRegex.FindStringSubmatch(header) + if len(tokens) != 2 { + log.Println("No token found in the header") + return false + } + key := fmt.Sprintf("token:%s", tokens[1]) + + var err error + + values, err := redis.Values(h.Redis.Do("HGETALL", key)) + if err == nil && len(values) > 0 { + if err = redis.ScanStruct(values, r); err == nil { + return true + } + } else { + log.Printf("Error while HGETALL %v\n", err) + } + + authorized := h.checkAuthToken(header, r) + authorized = true + + if authorized { + fmt.Printf("Token response: %#v\n", r) + _, err = h.Redis.Do("HMSET", redis.Args{}.Add(key).AddFlat(r)...) + if err != nil { + log.Printf("Error while setting token: %v\n", err) + return authorized + } + _, err = h.Redis.Do("EXPIRE", key, uint64(10*time.Minute/time.Second)) + if err != nil { + log.Printf("Error while setting expire on token: %v\n", err) + log.Println("Deleting token") + _, err = h.Redis.Do("DEL", key) + if err != nil { + log.Printf("Deleting token failed: %v", err) + } + return authorized + } + } + + return authorized +} + +func (h *microsubHandler) checkAuthToken(header string, token *TokenResponse) bool { + log.Println("Checking auth token") + req, err := http.NewRequest("GET", "https://publog.stuifzandapp.com/authtoken", nil) + if err != nil { + log.Println(err) + return false + } + + req.Header.Add("Authorization", header) + req.Header.Add("Accept", "application/json") + + client := http.Client{} + res, err := client.Do(req) + if err != nil { + log.Println(err) + return false + } + defer res.Body.Close() + + if res.StatusCode < 200 || res.StatusCode >= 300 { + log.Printf("HTTP StatusCode when verifying token: %d\n", res.StatusCode) + return false + } + + dec := json.NewDecoder(res.Body) + err = dec.Decode(&token) + + if err != nil { + log.Printf("Error in json object: %v", err) + return false + } + + log.Println("Auth Token: Success") + return true +} diff --git a/cmd/server/fetch.go b/cmd/server/fetch.go index 08908af..15d411c 100644 --- a/cmd/server/fetch.go +++ b/cmd/server/fetch.go @@ -64,7 +64,7 @@ func Fetch2(fetchURL string) (*microformats.Data, error) { log.Printf("MISS %s\n", u.String()) } - resp, err := http.Get(u.String()) + resp, err := http.Head(u.String()) if err != nil { return nil, fmt.Errorf("error while fetching %s: %s", u, err) } diff --git a/cmd/server/incoming.go b/cmd/server/incoming.go index 9e0529e..df6dd07 100644 --- a/cmd/server/incoming.go +++ b/cmd/server/incoming.go @@ -14,8 +14,9 @@ import ( // HubBackend handles information for the incoming handler type HubBackend interface { - CreateFeed(url string) int64 + CreateFeed(url string, contentType string) (int64, error) GetSecret(id int64) string + UpdateFeed(feedID int64, contentType string, content []byte) error } type incomingHandler struct { @@ -93,12 +94,17 @@ func (h *incomingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { ct := r.Header.Get("Content-Type") if strings.HasPrefix(ct, "application/rss+xml") { // RSS parsing + h.Backend.UpdateFeed(feed, ct, feedContent) } else if strings.HasPrefix(ct, "application/atom+xml") { // Atom parsing + h.Backend.UpdateFeed(feed, ct, feedContent) } else if strings.HasPrefix(ct, "text/html") { // h-entry parsing + h.Backend.UpdateFeed(feed, ct, feedContent) } else { - http.Error(w, "Unknown format of body", 400) + http.Error(w, fmt.Sprintf("Unknown format of body: %s", ct), 400) return } + + return } diff --git a/cmd/server/main.go b/cmd/server/main.go index 0be39ab..73c42ba 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -27,12 +27,12 @@ import ( "net/url" "os" "regexp" - "strings" "time" + "rss" + "github.com/garyburd/redigo/redis" "github.com/pstuifzand/microsub-server/microsub" - "willnorris.com/go/microformats" ) var ( @@ -74,19 +74,21 @@ func (h *hubIncomingBackend) GetSecret(id int64) string { return secret } -var hubUrl = "https://hub.stuifzandapp.com/" +var hubURL = "https://hub.stuifzandapp.com/" -func (h *hubIncomingBackend) CreateFeed(topic string) int64 { +func (h *hubIncomingBackend) CreateFeed(topic string, contentType string) (int64, error) { id, err := redis.Int64(h.conn.Do("INCR", "feed:next_id")) + if err != nil { - log.Println(err) + return 0, err } h.conn.Do("HSET", fmt.Sprintf("feed:%d", id), "url", topic) + h.conn.Do("HSET", fmt.Sprintf("feed:%d", id), "type", contentType) secret := randStringBytes(16) h.conn.Do("HSET", fmt.Sprintf("feed:%d", id), "secret", secret) - hub, err := url.Parse(hubUrl) + hub, err := url.Parse(hubURL) q := hub.Query() q.Add("hub.mode", "subscribe") q.Add("hub.callback", fmt.Sprintf("https://microsub.stuifzandapp.com/incoming/%d", id)) @@ -99,211 +101,87 @@ func (h *hubIncomingBackend) CreateFeed(topic string) int64 { res, err := client.PostForm(hub.String(), q) if err != nil { log.Printf("new request: %s\n", err) - return -1 + return 0, err } defer res.Body.Close() - fmt.Println(res) + filename := fmt.Sprintf("backend/feeds/%d.json", id) - return id -} - -func simplify(itemType string, item map[string][]interface{}) map[string]interface{} { - feedItem := make(map[string]interface{}) - - for k, v := range item { - if k == "bookmark-of" || k == "like-of" || k == "repost-of" || k == "in-reply-to" { - if value, ok := v[0].(*microformats.Microformat); ok { - - mType := value.Type[0][2:] - m := simplify(mType, value.Properties) - m["type"] = mType - feedItem[k] = []interface{}{m} - } else { - feedItem[k] = v - } - } else if k == "content" { - if content, ok := v[0].(map[string]interface{}); ok { - if text, e := content["value"]; e { - delete(content, "value") - content["text"] = text - } - feedItem[k] = content - } - } else if k == "photo" { - if itemType == "card" { - if len(v) >= 1 { - if value, ok := v[0].(string); ok { - feedItem[k] = value - } - } - } else { - feedItem[k] = v - } - } else if k == "video" { - feedItem[k] = v - } else if k == "featured" { - feedItem[k] = v - } else if value, ok := v[0].(*microformats.Microformat); ok { - mType := value.Type[0][2:] - m := simplify(mType, value.Properties) - m["type"] = mType - feedItem[k] = m - } else if value, ok := v[0].(string); ok { - feedItem[k] = value - } else if value, ok := v[0].(map[string]interface{}); ok { - feedItem[k] = value - } else if value, ok := v[0].([]interface{}); ok { - feedItem[k] = value - } - } - - // Remove "name" when it's equals to "content[text]" - if name, e := feedItem["name"]; e { - if content, e2 := feedItem["content"]; e2 { - if contentMap, ok := content.(map[string]interface{}); ok { - if text, e3 := contentMap["text"]; e3 { - if strings.TrimSpace(name.(string)) == strings.TrimSpace(text.(string)) { - delete(feedItem, "name") - } - } - } - } - } - - return feedItem -} - -func simplifyMicroformat(item *microformats.Microformat) map[string]interface{} { - itemType := item.Type[0][2:] - newItem := simplify(itemType, item.Properties) - newItem["type"] = itemType - - children := []map[string]interface{}{} - - if len(item.Children) > 0 { - for _, c := range item.Children { - child := simplifyMicroformat(c) - if c, e := child["children"]; e { - if ar, ok := c.([]map[string]interface{}); ok { - children = append(children, ar...) - } - delete(child, "children") - } - children = append(children, child) - } - - newItem["children"] = children - } - - return newItem -} - -func simplifyMicroformatData(md *microformats.Data) []map[string]interface{} { - items := []map[string]interface{}{} - for _, item := range md.Items { - newItem := simplifyMicroformat(item) - items = append(items, newItem) - if c, e := newItem["children"]; e { - if ar, ok := c.([]map[string]interface{}); ok { - items = append(items, ar...) - } - delete(newItem, "children") - } - } - return items -} - -// TokenResponse is the information that we get back from the token endpoint of the user... -type TokenResponse struct { - Me string `json:"me"` - ClientID string `json:"client_id"` - Scope string `json:"scope"` - IssuedAt int64 `json:"issued_at"` - Nonce int64 `json:"nonce"` -} - -var authHeaderRegex = regexp.MustCompile("^Bearer (.+)$") - -func (h *microsubHandler) cachedCheckAuthToken(header string, r *TokenResponse) bool { - log.Println("Cached checking Auth Token") - - tokens := authHeaderRegex.FindStringSubmatch(header) - if len(tokens) != 2 { - log.Println("No token found in the header") - return false - } - key := fmt.Sprintf("token:%s", tokens[1]) - - var err error - - values, err := redis.Values(h.Redis.Do("HGETALL", key)) - if err == nil && len(values) > 0 { - if err = redis.ScanStruct(values, r); err == nil { - return true - } - } else { - log.Printf("Error while HGETALL %v\n", err) - } - - authorized := h.checkAuthToken(header, r) - - if authorized { - fmt.Printf("Token response: %#v\n", r) - _, err = h.Redis.Do("HMSET", redis.Args{}.Add(key).AddFlat(r)...) - if err != nil { - log.Printf("Error while setting token: %v\n", err) - return authorized - } - _, err = h.Redis.Do("EXPIRE", key, uint64(10*time.Minute/time.Second)) - if err != nil { - log.Printf("Error while setting expire on token: %v\n", err) - log.Println("Deleting token") - _, err = h.Redis.Do("DEL", key) - if err != nil { - log.Printf("Deleting token failed: %v", err) - } - return authorized - } - } - - return authorized -} - -func (h *microsubHandler) checkAuthToken(header string, token *TokenResponse) bool { - log.Println("Checking auth token") - req, err := http.NewRequest("GET", "https://publog.stuifzandapp.com/authtoken", nil) + feed, err := rss.Fetch(topic) if err != nil { - log.Println(err) - return false + return 0, err } + os.MkdirAll("backend/feeds", 0755) - req.Header.Add("Authorization", header) - req.Header.Add("Accept", "application/json") - - client := http.Client{} - res, err := client.Do(req) + f, err := os.Create(filename) if err != nil { - log.Println(err) - return false - } - defer res.Body.Close() - - if res.StatusCode < 200 || res.StatusCode >= 300 { - log.Printf("HTTP StatusCode when verifying token: %d\n", res.StatusCode) - return false + return 0, err } - dec := json.NewDecoder(res.Body) - err = dec.Decode(&token) + defer f.Close() + out := json.NewEncoder(f) + err = out.Encode(feed) if err != nil { - log.Printf("Error in json object: %v", err) - return false + return 0, err } - log.Println("Auth Token: Success") - return true + return id, nil +} + +func readFeedFile(filename string) (*rss.Feed, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + var feed *rss.Feed + out := json.NewDecoder(f) + err = out.Decode(feed) + if err != nil { + return nil, err + } + return feed, err +} + +func writeFeedFile(filename string, feed *rss.Feed) error { + f, err := os.Create(filename) + if err != nil { + return err + } + + defer f.Close() + + out := json.NewEncoder(f) + err = out.Encode(feed) + if err != nil { + return err + } + + return nil +} + +func (h *hubIncomingBackend) UpdateFeed(feedID int64, contentType string, content []byte) error { + _, err := redis.String(h.conn.Do("HGET", fmt.Sprintf("feed:%d", feedID), "url")) + if err != nil { + return err + } + + filename := fmt.Sprintf("backend/feeds/%d.json", feedID) + + feed, err := readFeedFile(filename) + if err != nil { + return err + } + + err = feed.UpdateWithContent(content) + if err != nil { + return err + } + + err = writeFeedFile(filename, feed) + + return err } func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -350,6 +228,7 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } else if action == "preview" { md, err := Fetch2(values.Get("url")) if err != nil { + log.Println(err) http.Error(w, "Failed parsing url", 500) return } @@ -404,7 +283,7 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } else if action == "follow" { uid := values.Get("channel") url := values.Get("url") - h.HubIncomingBackend.CreateFeed(url) + h.HubIncomingBackend.CreateFeed(url, "application/rss+xml") feed := h.Backend.FollowURL(uid, url) w.Header().Add("Content-Type", "application/json") jw := json.NewEncoder(w) @@ -487,10 +366,11 @@ func main() { if createBackend { backend = createMemoryBackend() - } else { - backend = loadMemoryBackend(conn) + return } + backend = loadMemoryBackend(conn) + hubBackend := hubIncomingBackend{conn} http.Handle("/microsub", µsubHandler{ diff --git a/cmd/server/memory.go b/cmd/server/memory.go index 8754f99..e30ec8b 100644 --- a/cmd/server/memory.go +++ b/cmd/server/memory.go @@ -130,10 +130,35 @@ func (b *memoryBackend) ChannelsDelete(uid string) { } } +func mapToItem(result map[string]interface{}) microsub.Item { + item := microsub.Item{} + + if name, e := result["name"]; e { + item.Name = name.(string) + } + + if content, e := result["content"]; e { + if c, ok := content.(map[string]interface{}); ok { + if html, e2 := c["html"]; e2 { + item.Content.HTML = html.(string) + } + if text, e2 := c["value"]; e2 { + item.Content.Text = text.(string) + } + } + } + + if published, e := result["published"]; e { + item.Published = published.(string) + } + + return item +} + func (b *memoryBackend) TimelineGet(after, before, channel string) microsub.Timeline { feeds := b.FollowGetList(channel) - items := []map[string]interface{}{} + items := []microsub.Item{} for _, feed := range feeds { md, err := Fetch2(feed.URL) @@ -191,15 +216,16 @@ func (b *memoryBackend) TimelineGet(after, before, channel string) microsub.Time } if _, e := r["published"]; e { - items = append(items, r) + item := mapToItem(r) + items = append(items, item) } } } } sort.SliceStable(items, func(a, b int) bool { - timeA, _ := items[a]["published"].(string) - timeB, _ := items[b]["published"].(string) + timeA := items[a].Published + timeB := items[b].Published return strings.Compare(timeA, timeB) > 0 }) @@ -282,8 +308,13 @@ func (b *memoryBackend) PreviewURL(previewURL string) microsub.Timeline { return microsub.Timeline{} } results := simplifyMicroformatData(md) + items := []microsub.Item{} + for _, r := range results { + item := mapToItem(r) + items = append(items, item) + } return microsub.Timeline{ - Items: results, + Items: items, } } diff --git a/cmd/server/null.go b/cmd/server/null.go index 4e6cbb7..ac18c90 100644 --- a/cmd/server/null.go +++ b/cmd/server/null.go @@ -59,7 +59,7 @@ func (b *NullBackend) ChannelsDelete(uid string) { func (b *NullBackend) TimelineGet(after, before, channel string) microsub.Timeline { return microsub.Timeline{ Paging: microsub.Pagination{}, - Items: []map[string]interface{}{}, + Items: []microsub.Item{}, } } @@ -81,7 +81,7 @@ func (b *NullBackend) Search(query string) []microsub.Feed { func (b *NullBackend) PreviewURL(url string) microsub.Timeline { return microsub.Timeline{ Paging: microsub.Pagination{}, - Items: []map[string]interface{}{}, + Items: []microsub.Item{}, } } diff --git a/cmd/server/simplify.go b/cmd/server/simplify.go new file mode 100644 index 0000000..10b1b78 --- /dev/null +++ b/cmd/server/simplify.go @@ -0,0 +1,113 @@ +package main + +import ( + "strings" + + "willnorris.com/go/microformats" +) + +func simplify(itemType string, item map[string][]interface{}) map[string]interface{} { + feedItem := make(map[string]interface{}) + + for k, v := range item { + if k == "bookmark-of" || k == "like-of" || k == "repost-of" || k == "in-reply-to" { + if value, ok := v[0].(*microformats.Microformat); ok { + + mType := value.Type[0][2:] + m := simplify(mType, value.Properties) + m["type"] = mType + feedItem[k] = []interface{}{m} + } else { + feedItem[k] = v + } + } else if k == "content" { + if content, ok := v[0].(map[string]interface{}); ok { + if text, e := content["value"]; e { + delete(content, "value") + content["text"] = text + } + feedItem[k] = content + } + } else if k == "photo" { + if itemType == "card" { + if len(v) >= 1 { + if value, ok := v[0].(string); ok { + feedItem[k] = value + } + } + } else { + feedItem[k] = v + } + } else if k == "video" { + feedItem[k] = v + } else if k == "featured" { + feedItem[k] = v + } else if value, ok := v[0].(*microformats.Microformat); ok { + mType := value.Type[0][2:] + m := simplify(mType, value.Properties) + m["type"] = mType + feedItem[k] = m + } else if value, ok := v[0].(string); ok { + feedItem[k] = value + } else if value, ok := v[0].(map[string]interface{}); ok { + feedItem[k] = value + } else if value, ok := v[0].([]interface{}); ok { + feedItem[k] = value + } + } + + // Remove "name" when it's equals to "content[text]" + if name, e := feedItem["name"]; e { + if content, e2 := feedItem["content"]; e2 { + if contentMap, ok := content.(map[string]interface{}); ok { + if text, e3 := contentMap["text"]; e3 { + if strings.TrimSpace(name.(string)) == strings.TrimSpace(text.(string)) { + delete(feedItem, "name") + } + } + } + } + } + + return feedItem +} + +func simplifyMicroformat(item *microformats.Microformat) map[string]interface{} { + itemType := item.Type[0][2:] + newItem := simplify(itemType, item.Properties) + newItem["type"] = itemType + + children := []map[string]interface{}{} + + if len(item.Children) > 0 { + for _, c := range item.Children { + child := simplifyMicroformat(c) + if c, e := child["children"]; e { + if ar, ok := c.([]map[string]interface{}); ok { + children = append(children, ar...) + } + delete(child, "children") + } + children = append(children, child) + } + + newItem["children"] = children + } + + return newItem +} + +func simplifyMicroformatData(md *microformats.Data) []map[string]interface{} { + items := []map[string]interface{}{} + for _, item := range md.Items { + newItem := simplifyMicroformat(item) + items = append(items, newItem) + if c, e := newItem["children"]; e { + if ar, ok := c.([]map[string]interface{}); ok { + items = append(items, ar...) + } + delete(newItem, "children") + } + } + return items +} diff --git a/microsub/protocol.go b/microsub/protocol.go index ad49158..2fe7b48 100644 --- a/microsub/protocol.go +++ b/microsub/protocol.go @@ -75,8 +75,8 @@ type Pagination struct { // Timeline is a combination of items and paging information type Timeline struct { - Items []map[string]interface{} `json:"items"` - Paging Pagination `json:"paging"` + Items []Item `json:"items"` + Paging Pagination `json:"paging"` } type Feed struct {