Move websub to own package

- Move websub own package
- Add flag to disable auth (by default enabled)
- Process feed directly when following

cmd/ek/main.go

@@ -12,6 +12,10 @@ import (
 	"github.com/pstuifzand/ekster/pkg/indieauth"
 )
 
+func init() {
+	log.SetFlags(log.Lshortfile | log.Ldate | log.Ltime)
+}
+
 func loadAuth(c *client.Client, filename string) error {
 	f, err := os.Open(filename)
 	if err != nil {

cmd/eksterd/main.go

@@ -23,27 +23,29 @@ import (
 	"io"
 	"log"
 	"net/http"
-	"net/url"
 	"os"
 	"regexp"
 	"time"
 
-	"linkheader"
-
 	"github.com/garyburd/redigo/redis"
 	"github.com/pstuifzand/ekster/microsub"
 	"github.com/pstuifzand/ekster/pkg/util"
+	"github.com/pstuifzand/ekster/pkg/websub"
 )
 
 var (
 	pool        *redis.Pool
 	port        int
+	auth        bool
 	redisServer = flag.String("redis", "redis:6379", "")
 	entryRegex  = regexp.MustCompile("^entry\\[\\d+\\]$")
 )
 
 func init() {
+	log.SetFlags(log.Lshortfile | log.Ldate | log.Ltime)
+
 	flag.IntVar(&port, "port", 80, "port for serving api")
+	flag.BoolVar(&auth, "auth", true, "use auth")
 }
 
 type mainHandler struct {
@@ -73,29 +75,6 @@ func (h *hubIncomingBackend) GetSecret(id int64) string {
 	return secret
 }
 
-func (h *hubIncomingBackend) getHubURL(topic string) (string, error) {
-	client := &http.Client{}
-
-	resp, err := client.Head(topic)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-
-	if headers, e := resp.Header["Link"]; e {
-		links := linkheader.ParseMultiple(headers)
-		for _, link := range links {
-			if link.Rel == "hub" {
-				log.Printf("WebSub Hub URL found for topic=%s hub=%s\n", topic, link.URL)
-				return link.URL, nil
-			}
-		}
-	}
-
-	log.Printf("WebSub Hub URL not found for topic=%s\n", topic)
-	return "", nil
-}
-
 func (h *hubIncomingBackend) CreateFeed(topic string, channel string) (int64, error) {
 	conn := pool.Get()
 	defer conn.Close()
@@ -110,29 +89,23 @@ func (h *hubIncomingBackend) CreateFeed(topic string, channel string) (int64, er
 	secret := util.RandStringBytes(16)
 	conn.Do("HSET", fmt.Sprintf("feed:%d", id), "secret", secret)
 
-	hubURL, err := h.getHubURL(topic)
+	client := &http.Client{}
+
+	hubURL, err := websub.GetHubURL(client, topic)
+	if hubURL == "" {
+		log.Printf("WebSub Hub URL not found for topic=%s\n", topic)
+	} else {
+		log.Printf("WebSub Hub URL found for topic=%s hub=%s\n", topic, hubURL)
+	}
+
 	if err == nil && hubURL != "" {
 		conn.Do("HSET", fmt.Sprintf("feed:%d", id), "hub", hubURL)
 	} else {
 		return id, nil
 	}
 
-	hub, err := url.Parse(hubURL)
+	callbackURL := fmt.Sprintf("%s/incoming/%d", os.Getenv("EKSTER_BASEURL"), id)
-	q := hub.Query()
+	websub.Subscribe(client, hubURL, topic, callbackURL, secret, 24*3600)
-	q.Add("hub.mode", "subscribe")
-	q.Add("hub.callback", fmt.Sprintf("%s/incoming/%d", os.Getenv("EKSTER_BASEURL"), id))
-	q.Add("hub.topic", topic)
-	q.Add("hub.secret", secret)
-	hub.RawQuery = ""
-
-	log.Printf("POST %s\n", hub)
-	client := &http.Client{}
-	res, err := client.PostForm(hub.String(), q)
-	if err != nil {
-		log.Printf("new request: %s\n", err)
-		return 0, err
-	}
-	defer res.Body.Close()
 
 	return id, nil
 }
@@ -168,6 +141,12 @@ func main() {
 	log.Println("eksterd - microsub server")
 	flag.Parse()
 
+	if auth {
+		log.Println("Using auth")
+	} else {
+		log.Println("Authentication disabled")
+	}
+
 	if _, e := os.LookupEnv("EKSTER_BASEURL"); !e {
 		log.Fatal("EKSTER_BASEURL environment variable not found, please set with external url: https://example.com")
 	}

cmd/eksterd/memory.go

@@ -50,7 +50,6 @@ type Debug interface {
 }
 
 func init() {
-	log.SetFlags(log.Lshortfile | log.Ldate | log.Ltime)
 }
 
 func (b *memoryBackend) Debug() {
@@ -550,7 +549,17 @@ func (b *memoryBackend) FollowGetList(uid string) []microsub.Feed {
 func (b *memoryBackend) FollowURL(uid string, url string) microsub.Feed {
 	defer b.save()
 	feed := microsub.Feed{Type: "feed", URL: url}
+
+	resp, err := b.Fetch3(uid, feed.URL)
+	if err != nil {
+		return feed
+	}
+	defer resp.Body.Close()
+
 	b.Feeds[uid] = append(b.Feeds[uid], feed)
+
+	b.ProcessContent(uid, feed.URL, resp.Header.Get("Content-Type"), resp.Body)
+
 	return feed
 }
 

cmd/eksterd/microsub.go

@@ -26,6 +26,8 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	log.Printf("%s %s\n", r.Method, r.URL)
 	log.Println(r.URL.Query())
 	log.Println(r.PostForm)
+
+	if auth {
 		authorization := r.Header.Get("Authorization")
 
 		var token TokenResponse
@@ -41,6 +43,7 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			http.Error(w, "Wrong me", 403)
 			return
 		}
+	}
 
 	if r.Method == http.MethodGet {
 		values := r.URL.Query()

pkg/websub/subscribe.go

@@ -0,0 +1,60 @@
+package websub
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+
+	"linkheader"
+)
+
+// Fetcher return the response for a url
+type Fetcher interface {
+	Fetch(url string) (*http.Response, error)
+}
+
+// GetHubURL finds the HubURL for topic
+func GetHubURL(client *http.Client, topic string) (string, error) {
+	resp, err := client.Head(topic)
+
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	if headers, e := resp.Header["Link"]; e {
+		links := linkheader.ParseMultiple(headers)
+		for _, link := range links {
+			if link.Rel == "hub" {
+				return link.URL, nil
+			}
+		}
+	}
+
+	return "", nil
+}
+
+// Subscribe subscribes topicURL on hubURL
+func Subscribe(client *http.Client, hubURL, topicURL, callbackURL, secret string, leaseSeconds int) error {
+	hub, err := url.Parse(hubURL)
+	if err != nil {
+		return err
+	}
+
+	q := hub.Query()
+	q.Add("hub.mode", "subscribe")
+	q.Add("hub.callback", callbackURL)
+	q.Add("hub.topic", topicURL)
+	q.Add("hub.secret", secret)
+	q.Add("hub.lease_seconds", fmt.Sprintf("%d", leaseSeconds))
+	hub.RawQuery = ""
+
+	res, err := client.PostForm(hub.String(), q)
+	if err != nil {
+		return err
+	}
+
+	defer res.Body.Close()
+
+	return nil
+}