peter/ekster
1
0
Fork 0
Code Issues 3 Pull Requests Projects 3 Releases 1 Wiki Activity

Add redis caching of http requests and hub subscription

Browse Source
This commit is contained in:
Peter Stuifzand 2018-05-03 21:50:16 +02:00
parent dd4b03c97b
commit e935820387
16 changed files with 556 additions and 181 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

174
cmd/server/fetch.go
View File

@ -433,6 +433,9 @@ type redisItem struct {
// Fetch2 fetches stuff
func Fetch2(fetchURL string) (*http.Response, error) {
conn := pool.Get()
defer conn.Close()
if !strings.HasPrefix(fetchURL, "http") {
return nil, fmt.Errorf("error parsing %s as url", fetchURL)
}
@ -444,18 +447,15 @@ func Fetch2(fetchURL string) (*http.Response, error) {
req, err := http.NewRequest("GET", u.String(), nil)
if data, e := cache[u.String()]; e {
if data.created.After(time.Now().Add(time.Hour * -1)) {
log.Printf("HIT %s - %s\n", u.String(), time.Now().Sub(data.created).String())
rd := bufio.NewReader(bytes.NewReader(data.item))
cacheKey := fmt.Sprintf("http_cache:%s", u.String())
data, err := redis.Bytes(conn.Do("GET", cacheKey))
if err == nil {
log.Printf("HIT %s\n", u.String())
rd := bufio.NewReader(bytes.NewReader(data))
return http.ReadResponse(rd, req)
} else {
log.Printf("EXPIRE %s\n", u.String())
delete(cache, u.String())
}
} else {
log.Printf("MISS %s\n", u.String())
}
client := http.Client{}
resp, err := client.Do(req)
@ -470,162 +470,8 @@ func Fetch2(fetchURL string) (*http.Response, error) {
cur := b.Bytes()
copy(cachedCopy, cur)
cache[u.String()] = cacheItem{item: cachedCopy, created: time.Now()}
conn.Do("SET", cacheKey, cachedCopy, "EX", 60*60)
cachedResp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(cachedCopy)), req)
return cachedResp, err
}
// func Fetch(fetchURL string) []microsub.Item {
// result := []microsub.Item{}
// if !strings.HasPrefix(fetchURL, "http") {
// return result
// }
// u, err := url.Parse(fetchURL)
// if err != nil {
// log.Printf("error parsing %s as url: %s", fetchURL, err)
// return result
// }
// resp, err := http.Get(u.String())
// if err != nil {
// log.Printf("error while fetching %s: %s", u, err)
// return result
// }
// if !strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") {
// log.Printf("Content Type of %s = %s", fetchURL, resp.Header.Get("Content-Type"))
// return result
// }
// defer resp.Body.Close()
// data := microformats.Parse(resp.Body, u)
// jw := json.NewEncoder(os.Stdout)
// jw.SetIndent("", " ")
// jw.Encode(data)
// author := &microsub.Card{}
// for _, item := range data.Items {
// if item.Type[0] == "h-feed" {
// for _, child := range item.Children {
// previewItem := convertMfToItem(child)
// result = append(result, previewItem)
// }
// } else if item.Type[0] == "h-card" {
// mf := item
// author.Filled = true
// author.Type = "card"
// for prop, value := range mf.Properties {
// switch prop {
// case "url":
// author.URL = value[0].(string)
// break
// case "name":
// author.Name = value[0].(string)
// break
// case "photo":
// author.Photo = value[0].(string)
// break
// default:
// fmt.Printf("prop name not implemented for author: %s with value %#v\n", prop, value)
// break
// }
// }
// } else if item.Type[0] == "h-entry" {
// previewItem := convertMfToItem(item)
// result = append(result, previewItem)
// }
// }
// for i, item := range result {
// if !item.Author.Filled {
// result[i].Author = author
// }
// }
// return result
// }
// func convertMfToItem(mf *microformats.Microformat) microsub.Item {
// item := microsub.Item{}
// item.Type = mf.Type[0]
// for prop, value := range mf.Properties {
// switch prop {
// case "published":
// item.Published = value[0].(string)
// break
// case "url":
// item.URL = value[0].(string)
// break
// case "name":
// item.Name = value[0].(string)
// break
// case "latitude":
// item.Latitude = value[0].(string)
// break
// case "longitude":
// item.Longitude = value[0].(string)
// break
// case "like-of":
// for _, v := range value {
// item.LikeOf = append(item.LikeOf, v.(string))
// }
// break
// case "bookmark-of":
// for _, v := range value {
// item.BookmarkOf = append(item.BookmarkOf, v.(string))
// }
// break
// case "in-reply-to":
// for _, v := range value {
// item.InReplyTo = append(item.InReplyTo, v.(string))
// }
// break
// case "summary":
// if content, ok := value[0].(map[string]interface{}); ok {
// item.Content.HTML = content["html"].(string)
// item.Content.Text = content["value"].(string)
// } else if content, ok := value[0].(string); ok {
// item.Content.Text = content
// }
// break
// case "photo":
// for _, v := range value {
// item.Photo = append(item.Photo, v.(string))
// }
// break
// case "category":
// for _, v := range value {
// item.Category = append(item.Category, v.(string))
// }
// break
// case "content":
// if content, ok := value[0].(map[string]interface{}); ok {
// item.Content.HTML = content["html"].(string)
// item.Content.Text = content["value"].(string)
// } else if content, ok := value[0].(string); ok {
// item.Content.Text = content
// }
// break
// default:
// fmt.Printf("prop name not implemented: %s with value %#v\n", prop, value)
// break
// }
// }
// if item.Name == strings.TrimSpace(item.Content.Text) {
// item.Name = ""
// }
// // TODO: for like name is the field that is set
// if item.Content.HTML == "" && len(item.LikeOf) > 0 {
// item.Name = ""
// }
// fmt.Printf("%#v\n", item)
// return item
// }

46
cmd/server/main.go
View File

@ -23,15 +23,17 @@ import (
"fmt"
"io"
"log"
"math/rand"
"net/http"
"net/url"
"os"
"regexp"
"time"
"linkheader"
"github.com/garyburd/redigo/redis"
"github.com/pstuifzand/microsub-server/microsub"
"github.com/pstuifzand/microsub-server/pkg/util"
)
var (
@ -55,16 +57,6 @@ type hubIncomingBackend struct {
backend *memoryBackend
}
const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
func randStringBytes(n int) string {
b := make([]byte, n)
for i := range b {
b[i] = letterBytes[rand.Intn(len(letterBytes))]
}
return string(b)
}
func (h *hubIncomingBackend) GetSecret(id int64) string {
conn := pool.Get()
defer conn.Close()
@ -75,7 +67,28 @@ func (h *hubIncomingBackend) GetSecret(id int64) string {
return secret
}
var hubURL = "https://hub.stuifzandapp.com/"
func (h *hubIncomingBackend) getHubURL(topic string) (string, error) {
client := &http.Client{}
resp, err := client.Head(topic)
if err != nil {
return "", err
}
defer resp.Body.Close()
if headers, e := resp.Header["Link"]; e {
links := linkheader.ParseMultiple(headers)
for _, link := range links {
if link.Rel == "hub" {
log.Printf("WebSub Hub URL found for topic=%s hub=%s\n", topic, link.URL)
return link.URL, nil
}
}
}
log.Printf("WebSub Hub URL not found for topic=%s\n", topic)
return "", nil
}
func (h *hubIncomingBackend) CreateFeed(topic string, channel string) (int64, error) {
conn := pool.Get()
@ -88,9 +101,16 @@ func (h *hubIncomingBackend) CreateFeed(topic string, channel string) (int64, er
conn.Do("HSET", fmt.Sprintf("feed:%d", id), "url", topic)
conn.Do("HSET", fmt.Sprintf("feed:%d", id), "channel", channel)
secret := randStringBytes(16)
secret := util.RandStringBytes(16)
conn.Do("HSET", fmt.Sprintf("feed:%d", id), "secret", secret)
hubURL, err := h.getHubURL(topic)
if err == nil && hubURL != "" {
conn.Do("HSET", fmt.Sprintf("feed:%d", id), "hub", hubURL)
} else {
return id, nil
}
hub, err := url.Parse(hubURL)
q := hub.Query()
q.Add("hub.mode", "subscribe")

5
cmd/server/memory.go
View File

@ -151,7 +151,7 @@ func (b *memoryBackend) ChannelsGetList() []microsub.Channel {
return channels
}
// ChannelsCreate creates no channels
// ChannelsCreate creates a channels
func (b *memoryBackend) ChannelsCreate(name string) microsub.Channel {
defer b.save()
uid := fmt.Sprintf("%04d", b.NextUid)
@ -166,7 +166,7 @@ func (b *memoryBackend) ChannelsCreate(name string) microsub.Channel {
return channel
}
// ChannelsUpdate updates no channels
// ChannelsUpdate updates a channels
func (b *memoryBackend) ChannelsUpdate(uid, name string) microsub.Channel {
defer b.save()
if c, e := b.Channels[uid]; e {
@ -177,6 +177,7 @@ func (b *memoryBackend) ChannelsUpdate(uid, name string) microsub.Channel {
return microsub.Channel{}
}
// ChannelsDelete deletes a channel
func (b *memoryBackend) ChannelsDelete(uid string) {
defer b.save()
if _, e := b.Channels[uid]; e {

15
pkg/util/randkey.go Normal file
View File

@ -0,0 +1,15 @@
package util
import (
"math/rand"
)
const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
func RandStringBytes(n int) string {
b := make([]byte, n)
for i := range b {
b[i] = letterBytes[rand.Intn(len(letterBytes))]
}
return string(b)
}

2
vendor/linkheader/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
cpu.out
linkheader.test

6
vendor/linkheader/.travis.yml vendored Normal file
View File

@ -0,0 +1,6 @@
language: go
go:
- 1.6
- 1.7
- tip

10
vendor/linkheader/CONTRIBUTING.mkd vendored Normal file
View File

@ -0,0 +1,10 @@
# Contributing
* Raise an issue if appropriate
* Fork the repo
* Bootstrap the dev dependencies (run `./script/bootstrap`)
* Make your changes
* Use [gofmt](https://golang.org/cmd/gofmt/)
* Make sure the tests pass (run `./script/test`)
* Make sure the linters pass (run `./script/lint`)
* Issue a pull request

21
vendor/linkheader/LICENSE vendored Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2016 Tom Hudson
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

35
vendor/linkheader/README.mkd vendored Normal file
View File

@ -0,0 +1,35 @@
# Golang Link Header Parser
Library for parsing HTTP Link headers. Requires Go 1.6 or higher.
Docs can be found on [the GoDoc page](https://godoc.org/github.com/tomnomnom/linkheader).
[![Build Status](https://travis-ci.org/tomnomnom/linkheader.svg)](https://travis-ci.org/tomnomnom/linkheader)
## Basic Example
```go
package main
import (
"fmt"
"github.com/tomnomnom/linkheader"
)
func main() {
header := "<https://api.github.com/user/58276/repos?page=2>; rel=\"next\"," +
"<https://api.github.com/user/58276/repos?page=2>; rel=\"last\""
links := linkheader.Parse(header)
for _, link := range links {
fmt.Printf("URL: %s; Rel: %s\n", link.URL, link.Rel)
}
}
// Output:
// URL: https://api.github.com/user/58276/repos?page=2; Rel: next
// URL: https://api.github.com/user/58276/repos?page=2; Rel: last
```

76
vendor/linkheader/examples_test.go vendored Normal file
View File

@ -0,0 +1,76 @@
package linkheader_test
import (
"fmt"
"github.com/tomnomnom/linkheader"
)
func ExampleParse() {
header := "<https://api.github.com/user/58276/repos?page=2>; rel=\"next\"," +
"<https://api.github.com/user/58276/repos?page=2>; rel=\"last\""
links := linkheader.Parse(header)
for _, link := range links {
fmt.Printf("URL: %s; Rel: %s\n", link.URL, link.Rel)
}
// Output:
// URL: https://api.github.com/user/58276/repos?page=2; Rel: next
// URL: https://api.github.com/user/58276/repos?page=2; Rel: last
}
func ExampleParseMultiple() {
headers := []string{
"<https://api.github.com/user/58276/repos?page=2>; rel=\"next\"",
"<https://api.github.com/user/58276/repos?page=2>; rel=\"last\"",
}
links := linkheader.ParseMultiple(headers)
for _, link := range links {
fmt.Printf("URL: %s; Rel: %s\n", link.URL, link.Rel)
}
// Output:
// URL: https://api.github.com/user/58276/repos?page=2; Rel: next
// URL: https://api.github.com/user/58276/repos?page=2; Rel: last
}
func ExampleLinks_FilterByRel() {
header := "<https://api.github.com/user/58276/repos?page=2>; rel=\"next\"," +
"<https://api.github.com/user/58276/repos?page=2>; rel=\"last\""
links := linkheader.Parse(header)
for _, link := range links.FilterByRel("last") {
fmt.Printf("URL: %s; Rel: %s\n", link.URL, link.Rel)
}
// Output:
// URL: https://api.github.com/user/58276/repos?page=2; Rel: last
}
func ExampleLink_String() {
link := linkheader.Link{
URL: "http://example.com/page/2",
Rel: "next",
}
fmt.Printf("Link: %s\n", link.String())
// Output:
// Link: <http://example.com/page/2>; rel="next"
}
func ExampleLinks_String() {
links := linkheader.Links{
{URL: "http://example.com/page/3", Rel: "next"},
{URL: "http://example.com/page/1", Rel: "last"},
}
fmt.Printf("Link: %s\n", links.String())
// Output:
// Link: <http://example.com/page/3>; rel="next", <http://example.com/page/1>; rel="last"
}

148
vendor/linkheader/main.go vendored Normal file
View File

@ -0,0 +1,148 @@
// Package linkheader provides functions for parsing HTTP Link headers
package linkheader
import (
"fmt"
"strings"
)
// A Link is a single URL and related parameters
type Link struct {
URL string
Rel string
Params map[string]string
}
// HasParam returns if a Link has a particular parameter or not
func (l Link) HasParam(key string) bool {
for p := range l.Params {
if p == key {
return true
}
}
return false
}
// Param returns the value of a parameter if it exists
func (l Link) Param(key string) string {
for k, v := range l.Params {
if key == k {
return v
}
}
return ""
}
// String returns the string representation of a link
func (l Link) String() string {
p := make([]string, 0, len(l.Params))
for k, v := range l.Params {
p = append(p, fmt.Sprintf("%s=\"%s\"", k, v))
}
if l.Rel != "" {
p = append(p, fmt.Sprintf("%s=\"%s\"", "rel", l.Rel))
}
return fmt.Sprintf("<%s>; %s", l.URL, strings.Join(p, "; "))
}
// Links is a slice of Link structs
type Links []Link
// FilterByRel filters a group of Links by the provided Rel attribute
func (l Links) FilterByRel(r string) Links {
links := make(Links, 0)
for _, link := range l {
if link.Rel == r {
links = append(links, link)
}
}
return links
}
// String returns the string representation of multiple Links
// for use in HTTP responses etc
func (l Links) String() string {
if l == nil {
return fmt.Sprint(nil)
}
var strs []string
for _, link := range l {
strs = append(strs, link.String())
}
return strings.Join(strs, ", ")
}
// Parse parses a raw Link header in the form:
// <url>; rel="foo", <url>; rel="bar"; wat="dis"
// returning a slice of Link structs
func Parse(raw string) Links {
var links Links
// One chunk: <url>; rel="foo"
for _, chunk := range strings.Split(raw, ",") {
link := Link{URL: "", Rel: "", Params: make(map[string]string)}
// Figure out what each piece of the chunk is
for _, piece := range strings.Split(chunk, ";") {
piece = strings.Trim(piece, " ")
if piece == "" {
continue
}
// URL
if piece[0] == '<' && piece[len(piece)-1] == '>' {
link.URL = strings.Trim(piece, "<>")
continue
}
// Params
key, val := parseParam(piece)
if key == "" {
continue
}
// Special case for rel
if strings.ToLower(key) == "rel" {
link.Rel = val
} else {
link.Params[key] = val
}
}
if link.URL != "" {
links = append(links, link)
}
}
return links
}
// ParseMultiple is like Parse, but accepts a slice of headers
// rather than just one header string
func ParseMultiple(headers []string) Links {
links := make(Links, 0)
for _, header := range headers {
links = append(links, Parse(header)...)
}
return links
}
// parseParam takes a raw param in the form key="val" and
// returns the key and value as seperate strings
func parseParam(raw string) (key, val string) {
parts := strings.SplitN(raw, "=", 2)
if len(parts) != 2 {
return "", ""
}
key = parts[0]
val = strings.Trim(parts[1], "\"")
return key, val
}

173
vendor/linkheader/main_test.go vendored Normal file
View File

@ -0,0 +1,173 @@
package linkheader
import "testing"
func TestSimple(t *testing.T) {
// Test case stolen from https://github.com/thlorenz/parse-link-header :)
header := "<https://api.github.com/user/9287/repos?page=3&per_page=100>; rel=\"next\", " +
"<https://api.github.com/user/9287/repos?page=1&per_page=100>; rel=\"prev\"; pet=\"cat\", " +
"<https://api.github.com/user/9287/repos?page=5&per_page=100>; rel=\"last\""
links := Parse(header)
if len(links) != 3 {
t.Errorf("Should have been 3 links returned, got %d", len(links))
}
if links[0].URL != "https://api.github.com/user/9287/repos?page=3&per_page=100" {
t.Errorf("First link should have URL 'https://api.github.com/user/9287/repos?page=3&per_page=100'")
}
if links[0].Rel != "next" {
t.Errorf("First link should have rel=\"next\"")
}
if len(links[0].Params) != 0 {
t.Errorf("First link should have exactly 0 params, but has %d", len(links[0].Params))
}
if len(links[1].Params) != 1 {
t.Errorf("Second link should have exactly 1 params, but has %d", len(links[1].Params))
}
if links[1].Params["pet"] != "cat" {
t.Errorf("Second link's 'pet' param should be 'cat', but was %s", links[1].Params["pet"])
}
}
func TestEmpty(t *testing.T) {
links := Parse("")
if links != nil {
t.Errorf("Return value should be nil, but was %s", len(links))
}
}
// Although not often seen in the wild, the grammar in RFC 5988 suggests that it's
// valid for a link header to have nothing but a URL.
func TestNoRel(t *testing.T) {
links := Parse("<http://example.com>")
if len(links) != 1 {
t.Fatalf("Length of links should be 1, but was %d", len(links))
}
if links[0].URL != "http://example.com" {
t.Errorf("URL should be http://example.com, but was %s", links[0].URL)
}
}
func TestLinkMethods(t *testing.T) {
header := "<https://api.github.com/user/9287/repos?page=1&per_page=100>; rel=\"prev\"; pet=\"cat\""
links := Parse(header)
link := links[0]
if link.HasParam("foo") {
t.Errorf("Link should not have param 'foo'")
}
val := link.Param("pet")
if val != "cat" {
t.Errorf("Link should have param pet=\"cat\"")
}
val = link.Param("foo")
if val != "" {
t.Errorf("Link should not have value for param 'foo'")
}
}
func TestLinksMethods(t *testing.T) {
header := "<https://api.github.com/user/9287/repos?page=3&per_page=100>; rel=\"next\", " +
"<https://api.github.com/user/9287/repos?page=1&per_page=100>; rel=\"stylesheet\"; pet=\"cat\", " +
"<https://api.github.com/user/9287/repos?page=5&per_page=100>; rel=\"stylesheet\""
links := Parse(header)
filtered := links.FilterByRel("next")
if filtered[0].URL != "https://api.github.com/user/9287/repos?page=3&per_page=100" {
t.Errorf("URL did not match expected")
}
filtered = links.FilterByRel("stylesheet")
if len(filtered) != 2 {
t.Errorf("Filter for stylesheet should yield 2 results but got %d", len(filtered))
}
filtered = links.FilterByRel("notarel")
if len(filtered) != 0 {
t.Errorf("Filter by non-existant rel should yeild no results")
}
}
func TestParseMultiple(t *testing.T) {
headers := []string{
"<https://api.github.com/user/58276/repos?page=2>; rel=\"next\"",
"<https://api.github.com/user/58276/repos?page=2>; rel=\"last\"",
}
links := ParseMultiple(headers)
if len(links) != 2 {
t.Errorf("Should have returned 2 links")
}
}
func TestLinkToString(t *testing.T) {
l := Link{
URL: "http://example.com/page/2",
Rel: "next",
Params: map[string]string{
"foo": "bar",
},
}
have := l.String()
parsed := Parse(have)
if len(parsed) != 1 {
t.Errorf("Expected only 1 link")
}
if parsed[0].URL != l.URL {
t.Errorf("Re-parsed link header should have matching URL, but has `%s`", parsed[0].URL)
}
if parsed[0].Rel != l.Rel {
t.Errorf("Re-parsed link header should have matching rel, but has `%s`", parsed[0].Rel)
}
if parsed[0].Param("foo") != "bar" {
t.Errorf("Re-parsed link header should have foo=\"bar\" but doesn't")
}
}
func TestLinksToString(t *testing.T) {
ls := Links{
{URL: "http://example.com/page/3", Rel: "next"},
{URL: "http://example.com/page/1", Rel: "last"},
}
have := ls.String()
want := "<http://example.com/page/3>; rel=\"next\", <http://example.com/page/1>; rel=\"last\""
if have != want {
t.Errorf("Want `%s`, have `%s`", want, have)
}
}
func BenchmarkParse(b *testing.B) {
header := "<https://api.github.com/user/9287/repos?page=3&per_page=100>; rel=\"next\", " +
"<https://api.github.com/user/9287/repos?page=1&per_page=100>; rel=\"prev\"; pet=\"cat\", " +
"<https://api.github.com/user/9287/repos?page=5&per_page=100>; rel=\"last\""
for i := 0; i < b.N; i++ {
_ = Parse(header)
}
}

6
vendor/linkheader/script/bootstrap vendored Executable file
View File

@ -0,0 +1,6 @@
#!/bin/sh
PROJDIR=$(cd `dirname $0`/.. && pwd)
echo "Installing gometalinter and linters..."
go get github.com/alecthomas/gometalinter
gometalinter --install

6
vendor/linkheader/script/lint vendored Executable file
View File

@ -0,0 +1,6 @@
#!/bin/sh
PROJDIR=$(cd `dirname $0`/.. && pwd)
cd ${PROJDIR}
go get
gometalinter

7
vendor/linkheader/script/profile vendored Executable file
View File

@ -0,0 +1,7 @@
#!/bin/sh
set -e
PROJDIR=$(cd `dirname $0`/.. && pwd)
cd ${PROJDIR}
go test -bench . -benchmem -cpuprofile cpu.out
go tool pprof linkheader.test cpu.out

3
vendor/linkheader/script/test vendored Executable file
View File

@ -0,0 +1,3 @@
#!/bin/sh
PROJDIR=$(cd `dirname $0`/.. && pwd)
cd $PROJDIR && go test