peter/ekster
1
0
Fork 0
Code Issues 3 Pull Requests Projects 3 Releases 1 Wiki Activity

Use items for instead of maps

Browse Source
This commit is contained in:
Peter Stuifzand 2018-04-07 01:27:27 +02:00
parent a189b9eade
commit f3036526bb
8 changed files with 348 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
cmd/server/auth.go Normal file
View File

@ -0,0 +1,106 @@
package main
import (
"encoding/json"
"fmt"
"log"
"net/http"
"regexp"
"time"
"github.com/garyburd/redigo/redis"
)
// TokenResponse is the information that we get back from the token endpoint of the user...
type TokenResponse struct {
Me string `json:"me"`
ClientID string `json:"client_id"`
Scope string `json:"scope"`
IssuedAt int64 `json:"issued_at"`
Nonce int64 `json:"nonce"`
}
var authHeaderRegex = regexp.MustCompile("^Bearer (.+)$")
func (h *microsubHandler) cachedCheckAuthToken(header string, r *TokenResponse) bool {
r.Me = "https://publog.stuifzandapp.com/"
log.Println("Cached checking Auth Token")
tokens := authHeaderRegex.FindStringSubmatch(header)
if len(tokens) != 2 {
log.Println("No token found in the header")
return false
}
key := fmt.Sprintf("token:%s", tokens[1])
var err error
values, err := redis.Values(h.Redis.Do("HGETALL", key))
if err == nil && len(values) > 0 {
if err = redis.ScanStruct(values, r); err == nil {
return true
}
} else {
log.Printf("Error while HGETALL %v\n", err)
}
authorized := h.checkAuthToken(header, r)
authorized = true
if authorized {
fmt.Printf("Token response: %#v\n", r)
_, err = h.Redis.Do("HMSET", redis.Args{}.Add(key).AddFlat(r)...)
if err != nil {
log.Printf("Error while setting token: %v\n", err)
return authorized
}
_, err = h.Redis.Do("EXPIRE", key, uint64(10*time.Minute/time.Second))
if err != nil {
log.Printf("Error while setting expire on token: %v\n", err)
log.Println("Deleting token")
_, err = h.Redis.Do("DEL", key)
if err != nil {
log.Printf("Deleting token failed: %v", err)
}
return authorized
}
}
return authorized
}
func (h *microsubHandler) checkAuthToken(header string, token *TokenResponse) bool {
log.Println("Checking auth token")
req, err := http.NewRequest("GET", "https://publog.stuifzandapp.com/authtoken", nil)
if err != nil {
log.Println(err)
return false
}
req.Header.Add("Authorization", header)
req.Header.Add("Accept", "application/json")
client := http.Client{}
res, err := client.Do(req)
if err != nil {
log.Println(err)
return false
}
defer res.Body.Close()
if res.StatusCode < 200 || res.StatusCode >= 300 {
log.Printf("HTTP StatusCode when verifying token: %d\n", res.StatusCode)
return false
}
dec := json.NewDecoder(res.Body)
err = dec.Decode(&token)
if err != nil {
log.Printf("Error in json object: %v", err)
return false
}
log.Println("Auth Token: Success")
return true
}

2
cmd/server/fetch.go
View File

@ -64,7 +64,7 @@ func Fetch2(fetchURL string) (*microformats.Data, error) {
log.Printf("MISS %s\n", u.String())
}
resp, err := http.Get(u.String())
resp, err := http.Head(u.String())
if err != nil {
return nil, fmt.Errorf("error while fetching %s: %s", u, err)
}

10
cmd/server/incoming.go
View File

@ -14,8 +14,9 @@ import (
// HubBackend handles information for the incoming handler
type HubBackend interface {
CreateFeed(url string) int64
CreateFeed(url string, contentType string) (int64, error)
GetSecret(id int64) string
UpdateFeed(feedID int64, contentType string, content []byte) error
}
type incomingHandler struct {
@ -93,12 +94,17 @@ func (h *incomingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ct := r.Header.Get("Content-Type")
if strings.HasPrefix(ct, "application/rss+xml") {
// RSS parsing
h.Backend.UpdateFeed(feed, ct, feedContent)
} else if strings.HasPrefix(ct, "application/atom+xml") {
// Atom parsing
h.Backend.UpdateFeed(feed, ct, feedContent)
} else if strings.HasPrefix(ct, "text/html") {
// h-entry parsing
h.Backend.UpdateFeed(feed, ct, feedContent)
} else {
http.Error(w, "Unknown format of body", 400)
http.Error(w, fmt.Sprintf("Unknown format of body: %s", ct), 400)
return
}
return
}

280
cmd/server/main.go
View File

@ -27,12 +27,12 @@ import (
"net/url"
"os"
"regexp"
"strings"
"time"
"rss"
"github.com/garyburd/redigo/redis"
"github.com/pstuifzand/microsub-server/microsub"
"willnorris.com/go/microformats"
)
var (
@ -74,19 +74,21 @@ func (h *hubIncomingBackend) GetSecret(id int64) string {
return secret
}
var hubUrl = "https://hub.stuifzandapp.com/"
var hubURL = "https://hub.stuifzandapp.com/"
func (h *hubIncomingBackend) CreateFeed(topic string) int64 {
func (h *hubIncomingBackend) CreateFeed(topic string, contentType string) (int64, error) {
id, err := redis.Int64(h.conn.Do("INCR", "feed:next_id"))
if err != nil {
log.Println(err)
return 0, err
}
h.conn.Do("HSET", fmt.Sprintf("feed:%d", id), "url", topic)
h.conn.Do("HSET", fmt.Sprintf("feed:%d", id), "type", contentType)
secret := randStringBytes(16)
h.conn.Do("HSET", fmt.Sprintf("feed:%d", id), "secret", secret)
hub, err := url.Parse(hubUrl)
hub, err := url.Parse(hubURL)
q := hub.Query()
q.Add("hub.mode", "subscribe")
q.Add("hub.callback", fmt.Sprintf("https://microsub.stuifzandapp.com/incoming/%d", id))
@ -99,211 +101,87 @@ func (h *hubIncomingBackend) CreateFeed(topic string) int64 {
res, err := client.PostForm(hub.String(), q)
if err != nil {
log.Printf("new request: %s\n", err)
return -1
return 0, err
}
defer res.Body.Close()
fmt.Println(res)
filename := fmt.Sprintf("backend/feeds/%d.json", id)
return id
}
func simplify(itemType string, item map[string][]interface{}) map[string]interface{} {
feedItem := make(map[string]interface{})
for k, v := range item {
if k == "bookmark-of" || k == "like-of" || k == "repost-of" || k == "in-reply-to" {
if value, ok := v[0].(*microformats.Microformat); ok {
mType := value.Type[0][2:]
m := simplify(mType, value.Properties)
m["type"] = mType
feedItem[k] = []interface{}{m}
} else {
feedItem[k] = v
}
} else if k == "content" {
if content, ok := v[0].(map[string]interface{}); ok {
if text, e := content["value"]; e {
delete(content, "value")
content["text"] = text
}
feedItem[k] = content
}
} else if k == "photo" {
if itemType == "card" {
if len(v) >= 1 {
if value, ok := v[0].(string); ok {
feedItem[k] = value
}
}
} else {
feedItem[k] = v
}
} else if k == "video" {
feedItem[k] = v
} else if k == "featured" {
feedItem[k] = v
} else if value, ok := v[0].(*microformats.Microformat); ok {
mType := value.Type[0][2:]
m := simplify(mType, value.Properties)
m["type"] = mType
feedItem[k] = m
} else if value, ok := v[0].(string); ok {
feedItem[k] = value
} else if value, ok := v[0].(map[string]interface{}); ok {
feedItem[k] = value
} else if value, ok := v[0].([]interface{}); ok {
feedItem[k] = value
}
}
// Remove "name" when it's equals to "content[text]"
if name, e := feedItem["name"]; e {
if content, e2 := feedItem["content"]; e2 {
if contentMap, ok := content.(map[string]interface{}); ok {
if text, e3 := contentMap["text"]; e3 {
if strings.TrimSpace(name.(string)) == strings.TrimSpace(text.(string)) {
delete(feedItem, "name")
}
}
}
}
}
return feedItem
}
func simplifyMicroformat(item *microformats.Microformat) map[string]interface{} {
itemType := item.Type[0][2:]
newItem := simplify(itemType, item.Properties)
newItem["type"] = itemType
children := []map[string]interface{}{}
if len(item.Children) > 0 {
for _, c := range item.Children {
child := simplifyMicroformat(c)
if c, e := child["children"]; e {
if ar, ok := c.([]map[string]interface{}); ok {
children = append(children, ar...)
}
delete(child, "children")
}
children = append(children, child)
}
newItem["children"] = children
}
return newItem
}
func simplifyMicroformatData(md *microformats.Data) []map[string]interface{} {
items := []map[string]interface{}{}
for _, item := range md.Items {
newItem := simplifyMicroformat(item)
items = append(items, newItem)
if c, e := newItem["children"]; e {
if ar, ok := c.([]map[string]interface{}); ok {
items = append(items, ar...)
}
delete(newItem, "children")
}
}
return items
}
// TokenResponse is the information that we get back from the token endpoint of the user...
type TokenResponse struct {
Me string `json:"me"`
ClientID string `json:"client_id"`
Scope string `json:"scope"`
IssuedAt int64 `json:"issued_at"`
Nonce int64 `json:"nonce"`
}
var authHeaderRegex = regexp.MustCompile("^Bearer (.+)$")
func (h *microsubHandler) cachedCheckAuthToken(header string, r *TokenResponse) bool {
log.Println("Cached checking Auth Token")
tokens := authHeaderRegex.FindStringSubmatch(header)
if len(tokens) != 2 {
log.Println("No token found in the header")
return false
}
key := fmt.Sprintf("token:%s", tokens[1])
var err error
values, err := redis.Values(h.Redis.Do("HGETALL", key))
if err == nil && len(values) > 0 {
if err = redis.ScanStruct(values, r); err == nil {
return true
}
} else {
log.Printf("Error while HGETALL %v\n", err)
}
authorized := h.checkAuthToken(header, r)
if authorized {
fmt.Printf("Token response: %#v\n", r)
_, err = h.Redis.Do("HMSET", redis.Args{}.Add(key).AddFlat(r)...)
if err != nil {
log.Printf("Error while setting token: %v\n", err)
return authorized
}
_, err = h.Redis.Do("EXPIRE", key, uint64(10*time.Minute/time.Second))
if err != nil {
log.Printf("Error while setting expire on token: %v\n", err)
log.Println("Deleting token")
_, err = h.Redis.Do("DEL", key)
if err != nil {
log.Printf("Deleting token failed: %v", err)
}
return authorized
}
}
return authorized
}
func (h *microsubHandler) checkAuthToken(header string, token *TokenResponse) bool {
log.Println("Checking auth token")
req, err := http.NewRequest("GET", "https://publog.stuifzandapp.com/authtoken", nil)
feed, err := rss.Fetch(topic)
if err != nil {
log.Println(err)
return false
return 0, err
}
os.MkdirAll("backend/feeds", 0755)
req.Header.Add("Authorization", header)
req.Header.Add("Accept", "application/json")
client := http.Client{}
res, err := client.Do(req)
f, err := os.Create(filename)
if err != nil {
log.Println(err)
return false
}
defer res.Body.Close()
if res.StatusCode < 200 || res.StatusCode >= 300 {
log.Printf("HTTP StatusCode when verifying token: %d\n", res.StatusCode)
return false
return 0, err
}
dec := json.NewDecoder(res.Body)
err = dec.Decode(&token)
defer f.Close()
out := json.NewEncoder(f)
err = out.Encode(feed)
if err != nil {
log.Printf("Error in json object: %v", err)
return false
return 0, err
}
log.Println("Auth Token: Success")
return true
return id, nil
}
func readFeedFile(filename string) (*rss.Feed, error) {
f, err := os.Open(filename)
if err != nil {
return nil, err
}
defer f.Close()
var feed *rss.Feed
out := json.NewDecoder(f)
err = out.Decode(feed)
if err != nil {
return nil, err
}
return feed, err
}
func writeFeedFile(filename string, feed *rss.Feed) error {
f, err := os.Create(filename)
if err != nil {
return err
}
defer f.Close()
out := json.NewEncoder(f)
err = out.Encode(feed)
if err != nil {
return err
}
return nil
}
func (h *hubIncomingBackend) UpdateFeed(feedID int64, contentType string, content []byte) error {
_, err := redis.String(h.conn.Do("HGET", fmt.Sprintf("feed:%d", feedID), "url"))
if err != nil {
return err
}
filename := fmt.Sprintf("backend/feeds/%d.json", feedID)
feed, err := readFeedFile(filename)
if err != nil {
return err
}
err = feed.UpdateWithContent(content)
if err != nil {
return err
}
err = writeFeedFile(filename, feed)
return err
}
func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@ -350,6 +228,7 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} else if action == "preview" {
md, err := Fetch2(values.Get("url"))
if err != nil {
log.Println(err)
http.Error(w, "Failed parsing url", 500)
return
}
@ -404,7 +283,7 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} else if action == "follow" {
uid := values.Get("channel")
url := values.Get("url")
h.HubIncomingBackend.CreateFeed(url)
h.HubIncomingBackend.CreateFeed(url, "application/rss+xml")
feed := h.Backend.FollowURL(uid, url)
w.Header().Add("Content-Type", "application/json")
jw := json.NewEncoder(w)
@ -487,10 +366,11 @@ func main() {
if createBackend {
backend = createMemoryBackend()
} else {
backend = loadMemoryBackend(conn)
return
}
backend = loadMemoryBackend(conn)
hubBackend := hubIncomingBackend{conn}
http.Handle("/microsub", &microsubHandler{

41
cmd/server/memory.go
View File

@ -130,10 +130,35 @@ func (b *memoryBackend) ChannelsDelete(uid string) {
}
}
func mapToItem(result map[string]interface{}) microsub.Item {
item := microsub.Item{}
if name, e := result["name"]; e {
item.Name = name.(string)
}
if content, e := result["content"]; e {
if c, ok := content.(map[string]interface{}); ok {
if html, e2 := c["html"]; e2 {
item.Content.HTML = html.(string)
}
if text, e2 := c["value"]; e2 {
item.Content.Text = text.(string)
}
}
}
if published, e := result["published"]; e {
item.Published = published.(string)
}
return item
}
func (b *memoryBackend) TimelineGet(after, before, channel string) microsub.Timeline {
feeds := b.FollowGetList(channel)
items := []map[string]interface{}{}
items := []microsub.Item{}
for _, feed := range feeds {
md, err := Fetch2(feed.URL)
@ -191,15 +216,16 @@ func (b *memoryBackend) TimelineGet(after, before, channel string) microsub.Time
}
if _, e := r["published"]; e {
items = append(items, r)
item := mapToItem(r)
items = append(items, item)
}
}
}
}
sort.SliceStable(items, func(a, b int) bool {
timeA, _ := items[a]["published"].(string)
timeB, _ := items[b]["published"].(string)
timeA := items[a].Published
timeB := items[b].Published
return strings.Compare(timeA, timeB) > 0
})
@ -282,8 +308,13 @@ func (b *memoryBackend) PreviewURL(previewURL string) microsub.Timeline {
return microsub.Timeline{}
}
results := simplifyMicroformatData(md)
items := []microsub.Item{}
for _, r := range results {
item := mapToItem(r)
items = append(items, item)
}
return microsub.Timeline{
Items: results,
Items: items,
}
}

4
cmd/server/null.go
View File

@ -59,7 +59,7 @@ func (b *NullBackend) ChannelsDelete(uid string) {
func (b *NullBackend) TimelineGet(after, before, channel string) microsub.Timeline {
return microsub.Timeline{
Paging: microsub.Pagination{},
Items: []map[string]interface{}{},
Items: []microsub.Item{},
}
}
@ -81,7 +81,7 @@ func (b *NullBackend) Search(query string) []microsub.Feed {
func (b *NullBackend) PreviewURL(url string) microsub.Timeline {
return microsub.Timeline{
Paging: microsub.Pagination{},
Items: []map[string]interface{}{},
Items: []microsub.Item{},
}
}

113
cmd/server/simplify.go Normal file
View File

@ -0,0 +1,113 @@
package main
import (
"strings"
"willnorris.com/go/microformats"
)
func simplify(itemType string, item map[string][]interface{}) map[string]interface{} {
feedItem := make(map[string]interface{})
for k, v := range item {
if k == "bookmark-of" || k == "like-of" || k == "repost-of" || k == "in-reply-to" {
if value, ok := v[0].(*microformats.Microformat); ok {
mType := value.Type[0][2:]
m := simplify(mType, value.Properties)
m["type"] = mType
feedItem[k] = []interface{}{m}
} else {
feedItem[k] = v
}
} else if k == "content" {
if content, ok := v[0].(map[string]interface{}); ok {
if text, e := content["value"]; e {
delete(content, "value")
content["text"] = text
}
feedItem[k] = content
}
} else if k == "photo" {
if itemType == "card" {
if len(v) >= 1 {
if value, ok := v[0].(string); ok {
feedItem[k] = value
}
}
} else {
feedItem[k] = v
}
} else if k == "video" {
feedItem[k] = v
} else if k == "featured" {
feedItem[k] = v
} else if value, ok := v[0].(*microformats.Microformat); ok {
mType := value.Type[0][2:]
m := simplify(mType, value.Properties)
m["type"] = mType
feedItem[k] = m
} else if value, ok := v[0].(string); ok {
feedItem[k] = value
} else if value, ok := v[0].(map[string]interface{}); ok {
feedItem[k] = value
} else if value, ok := v[0].([]interface{}); ok {
feedItem[k] = value
}
}
// Remove "name" when it's equals to "content[text]"
if name, e := feedItem["name"]; e {
if content, e2 := feedItem["content"]; e2 {
if contentMap, ok := content.(map[string]interface{}); ok {
if text, e3 := contentMap["text"]; e3 {
if strings.TrimSpace(name.(string)) == strings.TrimSpace(text.(string)) {
delete(feedItem, "name")
}
}
}
}
}
return feedItem
}
func simplifyMicroformat(item *microformats.Microformat) map[string]interface{} {
itemType := item.Type[0][2:]
newItem := simplify(itemType, item.Properties)
newItem["type"] = itemType
children := []map[string]interface{}{}
if len(item.Children) > 0 {
for _, c := range item.Children {
child := simplifyMicroformat(c)
if c, e := child["children"]; e {
if ar, ok := c.([]map[string]interface{}); ok {
children = append(children, ar...)
}
delete(child, "children")
}
children = append(children, child)
}
newItem["children"] = children
}
return newItem
}
func simplifyMicroformatData(md *microformats.Data) []map[string]interface{} {
items := []map[string]interface{}{}
for _, item := range md.Items {
newItem := simplifyMicroformat(item)
items = append(items, newItem)
if c, e := newItem["children"]; e {
if ar, ok := c.([]map[string]interface{}); ok {
items = append(items, ar...)
}
delete(newItem, "children")
}
}
return items
}

4
microsub/protocol.go
View File

@ -75,8 +75,8 @@ type Pagination struct {
// Timeline is a combination of items and paging information
type Timeline struct {
Items []map[string]interface{} `json:"items"`
Paging Pagination `json:"paging"`
Items []Item `json:"items"`
Paging Pagination `json:"paging"`
}
type Feed struct {