Handle error from ProcessContent in HubBackend

- extract env vars
- add command line arguments
- move null backend to server package

cmd/eksterd/http.go

@@ -39,6 +39,7 @@ import (
 
 type mainHandler struct {
 	Backend     *memoryBackend
+	BaseURL     string
 	TemplateDir string
 }
 
@@ -101,13 +102,10 @@ type authRequest struct {
 	AccessToken string `redis:"access_token"`
 }
 
-func newMainHandler(backend *memoryBackend) (*mainHandler, error) {
+func newMainHandler(backend *memoryBackend, baseURL, templateDir string) (*mainHandler, error) {
 	h := &mainHandler{Backend: backend}
 
-	templateDir := os.Getenv("EKSTER_TEMPLATES")
+	h.BaseURL = baseURL
-	if templateDir == "" {
-		return nil, fmt.Errorf("missing env var EKSTER_TEMPLATES")
-	}
 
 	templateDir = strings.TrimRight(templateDir, "/")
 	h.TemplateDir = templateDir
@@ -211,7 +209,7 @@ func isLoggedIn(backend *memoryBackend, sess *session) bool {
 		return false
 	}
 
-	if !authEnabled {
+	if !backend.AuthEnabled {
 		return true
 	}
 
@@ -297,7 +295,7 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 			var page indexPage
 			page.Session = sess
-			page.Baseurl = strings.TrimRight(os.Getenv("EKSTER_BASEURL"), "/")
+			page.Baseurl = strings.TrimRight(h.BaseURL, "/")
 
 			err = h.renderTemplate(w, "index.html", page)
 			if err != nil {
@@ -517,7 +515,7 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			log.Println(authURL)
 
 			state := util.RandStringBytes(16)
-			redirectURI := fmt.Sprintf("%s/session/callback", os.Getenv("EKSTER_BASEURL"))
+			redirectURI := fmt.Sprintf("%s/session/callback", h.BaseURL)
 
 			sess, err := loadSession(sessionVar, conn)
 

cmd/eksterd/hubbackend.go

@@ -48,6 +48,7 @@ type HubBackend interface {
 
 type hubIncomingBackend struct {
 	backend *memoryBackend
+	baseURL string
 }
 
 func (h *hubIncomingBackend) GetSecret(id int64) string {
@@ -85,7 +86,7 @@ func (h *hubIncomingBackend) CreateFeed(topic string, channel string) (int64, er
 		log.Printf("WebSub Hub URL found for topic=%s hub=%s\n", topic, hubURL)
 	}
 
-	callbackURL := fmt.Sprintf("%s/incoming/%d", os.Getenv("EKSTER_BASEURL"), id)
+	callbackURL := fmt.Sprintf("%s/incoming/%d", h.baseURL, id)
 
 	if err == nil && hubURL != "" {
 		args := redis.Args{}.Add(fmt.Sprintf("feed:%d", id), "hub", hubURL, "callback", callbackURL)
@@ -112,8 +113,11 @@ func (h *hubIncomingBackend) UpdateFeed(feedID int64, contentType string, body i
 		return err
 	}
 
-	log.Printf("updating feed %d - %s %s\n", feedID, u, channel)
+	log.Printf("Updating feed %d - %s %s\n", feedID, u, channel)
-	h.backend.ProcessContent(channel, u, contentType, body)
+	err = h.backend.ProcessContent(channel, u, contentType, body)
+	if err != nil {
+		log.Printf("Error while updating content for channel %s: %s", channel, err)
+	}
 
 	return err
 }

cmd/eksterd/main.go

@@ -28,7 +28,6 @@ import (
 	"github.com/gomodule/redigo/redis"
 	"p83.nl/go/ekster/pkg/auth"
 
-	"p83.nl/go/ekster/pkg/microsub"
 	"p83.nl/go/ekster/pkg/server"
 )
 
@@ -36,18 +35,21 @@ const (
 	ClientID string = "https://p83.nl/microsub-client"
 )
 
+type AppOptions struct {
+	Port        int
+	AuthEnabled bool
+	Headless    bool
+	RedisServer string
+	BaseURL     string
+	TemplateDir string
+}
+
 var (
-	pool        *redis.Pool
+	pool *redis.Pool
-	port        int
-	authEnabled bool
-	redisServer = flag.String("redis", "redis:6379", "")
 )
 
 func init() {
 	log.SetFlags(log.Lshortfile | log.Ldate | log.Ltime)
-
-	flag.IntVar(&port, "port", 80, "port for serving api")
-	flag.BoolVar(&authEnabled, "auth", true, "use auth")
 }
 
 func newPool(addr string) *redis.Pool {
@@ -85,18 +87,93 @@ func WithAuth(handler http.Handler, b *memoryBackend) http.Handler {
 	})
 }
 
+type App struct {
+	options    AppOptions
+	backend    *memoryBackend
+	hubBackend *hubIncomingBackend
+}
+
+func (app *App) Run() {
+	app.backend.run()
+	app.hubBackend.run()
+
+	log.Printf("Listening on port %d\n", app.options.Port)
+	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", app.options.Port), nil))
+
+}
+
+func NewApp(options AppOptions) *App {
+	app := &App{
+		options: options,
+	}
+
+	var backend *memoryBackend
+
+	app.backend = loadMemoryBackend()
+	app.backend.AuthEnabled = options.AuthEnabled
+
+	app.hubBackend = &hubIncomingBackend{app.backend, options.BaseURL}
+
+	http.Handle("/micropub", &micropubHandler{
+		Backend: app.backend,
+	})
+
+	handler := server.NewMicrosubHandler(app.backend)
+	if options.AuthEnabled {
+		handler = WithAuth(handler, app.backend)
+	}
+
+	http.Handle("/microsub", handler)
+
+	http.Handle("/incoming/", &incomingHandler{
+		Backend: app.hubBackend,
+	})
+
+	if !options.Headless {
+		handler, err := newMainHandler(backend, options.BaseURL, options.TemplateDir)
+		if err != nil {
+			log.Fatal(err)
+		}
+		http.Handle("/", handler)
+	}
+
+	return app
+}
+
 func main() {
 	log.Println("eksterd - microsub server")
+
+	var options AppOptions
+
+	flag.IntVar(&options.Port, "port", 80, "port for serving api")
+	flag.BoolVar(&options.AuthEnabled, "auth", true, "use auth")
+	flag.BoolVar(&options.Headless, "headless", false, "disable frontend")
+	flag.StringVar(&options.RedisServer, "redis", "redis:6379", "redis server")
+	flag.StringVar(&options.BaseURL, "baseurl", "", "http server baseurl")
+	flag.StringVar(&options.TemplateDir, "templates", "./templates", "template directory")
+
 	flag.Parse()
 
-	if authEnabled {
+	if options.AuthEnabled {
 		log.Println("Using auth")
 	} else {
 		log.Println("Authentication disabled")
 	}
 
-	if _, e := os.LookupEnv("EKSTER_BASEURL"); !e {
+	if options.BaseURL == "" {
-		log.Fatal("EKSTER_BASEURL environment variable not found, please set with external url: https://example.com")
+		if envVar, e := os.LookupEnv("EKSTER_BASEURL"); e {
+			options.BaseURL = envVar
+		} else {
+			log.Fatal("EKSTER_BASEURL environment variable not found, please set with external url, -baseurl url option")
+		}
+	}
+
+	if options.TemplateDir == "" {
+		if envVar, e := os.LookupEnv("EKSTER_TEMPLATES"); e {
+			options.TemplateDir = envVar
+		} else {
+			log.Fatal("EKSTER_TEMPLATES environment variable not found, use env var or -templates dir option")
+		}
 	}
 
 	createBackend := false
@@ -108,13 +185,10 @@ func main() {
 		}
 	}
 
-	pool = newPool(*redisServer)
-
-	var backend microsub.Microsub
-
 	if createBackend {
-		backend = createMemoryBackend()
+		createMemoryBackend()
 
+		// TODO(peter): automatically gather this information from login or otherwise
 		log.Println(`Config file "backend.json" is created in the current directory.`)
 		log.Println(`Update "Me" variable to your website address "https://example.com/"`)
 		log.Println(`Update "TokenEndpoint" variable to the address of your token endpoint "https://example.com/token"`)
@@ -122,33 +196,7 @@ func main() {
 		return
 	}
 
-	backend = loadMemoryBackend()
+	pool = newPool(options.RedisServer)
 
-	hubBackend := hubIncomingBackend{backend.(*memoryBackend)}
+	NewApp(options).Run()
-
-	http.Handle("/micropub", &micropubHandler{
-		Backend: backend.(*memoryBackend),
-	})
-
-	handler := server.NewMicrosubHandler(backend)
-	if authEnabled {
-		handler = WithAuth(handler, backend.(*memoryBackend))
-	}
-
-	http.Handle("/microsub", handler)
-
-	http.Handle("/incoming/", &incomingHandler{
-		Backend: &hubBackend,
-	})
-	handler, err := newMainHandler(backend.(*memoryBackend))
-	if err != nil {
-		log.Fatal(err)
-	}
-	http.Handle("/", handler)
-
-	backend.(*memoryBackend).run()
-	hubBackend.run()
-
-	log.Printf("Listening on port %d\n", port)
-	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), nil))
 }

cmd/eksterd/memory.go

@@ -54,6 +54,7 @@ type memoryBackend struct {
 
 	Me            string
 	TokenEndpoint string
+	AuthEnabled   bool
 
 	ticker *time.Ticker
 	quit   chan struct{}
@@ -141,7 +142,7 @@ func (b *memoryBackend) save() {
 	jw.Encode(b)
 }
 
-func loadMemoryBackend() microsub.Microsub {
+func loadMemoryBackend() *memoryBackend {
 	backend := &memoryBackend{}
 	err := backend.load()
 	if err != nil {
@@ -153,7 +154,7 @@ func loadMemoryBackend() microsub.Microsub {
 	return backend
 }
 
-func createMemoryBackend() microsub.Microsub {
+func createMemoryBackend() {
 	backend := memoryBackend{}
 	backend.lock.Lock()
 
@@ -174,7 +175,6 @@ func createMemoryBackend() microsub.Microsub {
 	backend.lock.Unlock()
 
 	backend.save()
-	return &backend
 }
 
 // ChannelsGetList gets channels
@@ -475,7 +475,8 @@ func getPossibleURLs(query string) []string {
 func (b *memoryBackend) Search(query string) ([]microsub.Feed, error) {
 	urls := getPossibleURLs(query)
 
-	var feeds []microsub.Feed
+	// needs to be like this, because we get a null result otherwise in the json output
+	feeds := []microsub.Feed{}
 
 	for _, u := range urls {
 		log.Println(u)

pkg/client/requests.go

@@ -324,13 +324,14 @@ func (c *Client) Search(query string) ([]microsub.Feed, error) {
 func (c *Client) MarkRead(channel string, uids []string) error {
 	args := make(map[string]string)
 	args["channel"] = channel
+	args["method"] = "mark_read"
 
 	data := url.Values{}
 	for _, uid := range uids {
 		data.Add("entry[]", uid)
 	}
 
-	res, err := c.microsubPostFormRequest("mark_read", args, data)
+	res, err := c.microsubPostFormRequest("timeline", args, data)
 	if err != nil {
 		return err
 	}

pkg/server/microsub.go

@@ -108,7 +108,7 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			cons := newConsumer(conn)
 			h.backend.AddEventListener(cons)
 		} else {
-			http.Error(w, fmt.Sprintf("unknown action %s\n", action), 500)
+			http.Error(w, fmt.Sprintf("unknown action %s\n", action), 400)
 			return
 		}
 		return
@@ -210,9 +210,8 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 			respondJSON(w, []string{})
 		} else {
-			http.Error(w, fmt.Sprintf("unknown action %s\n", action), 500)
+			http.Error(w, fmt.Sprintf("unknown action %s\n", action), 400)
 		}
-
 		return
 	}
 	return

pkg/server/microsub_test.go

@@ -0,0 +1,170 @@
+package server
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"p83.nl/go/ekster/pkg/client"
+)
+
+func createServerClient() (*httptest.Server, *client.Client) {
+	backend := &NullBackend{}
+
+	handler := NewMicrosubHandler(backend)
+
+	server := httptest.NewServer(handler)
+
+	c := client.Client{
+		Logging: false,
+		Token:   "1234",
+	}
+
+	c.Me, _ = url.Parse("https://example.com/")
+	c.MicrosubEndpoint, _ = url.Parse(server.URL + "/microsub")
+
+	return server, &c
+}
+
+func TestServer_ChannelsGetList(t *testing.T) {
+	server, c := createServerClient()
+	defer server.Close()
+
+	channels, err := c.ChannelsGetList()
+
+	if assert.NoError(t, err) {
+		assert.Equal(t, 2, len(channels), "should return 2 channels")
+
+		assert.Equal(t, "notifications", channels[0].Name)
+		assert.Equal(t, "0001", channels[0].UID)
+		assert.Equal(t, 0, channels[0].Unread)
+
+		assert.Equal(t, "default", channels[1].Name)
+		assert.Equal(t, "0000", channels[1].UID)
+		assert.Equal(t, 0, channels[1].Unread)
+	}
+}
+
+func TestServer_ChannelsCreate(t *testing.T) {
+	server, c := createServerClient()
+	defer server.Close()
+
+	channel, err := c.ChannelsCreate("test")
+
+	if assert.NoError(t, err) {
+		assert.Equal(t, "test", channel.Name)
+	}
+}
+
+func TestServer_ChannelsDelete(t *testing.T) {
+	server, c := createServerClient()
+	defer server.Close()
+	err := c.ChannelsDelete("0001")
+	assert.NoError(t, err)
+}
+
+func TestServer_ChannelsUpdate(t *testing.T) {
+	server, c := createServerClient()
+	defer server.Close()
+	_, err := c.ChannelsUpdate("0001", "newname")
+	assert.NoError(t, err)
+}
+
+func TestServer_TimelineGet(t *testing.T) {
+	server, c := createServerClient()
+	defer server.Close()
+	timeline, err := c.TimelineGet("", "", "0001")
+	if assert.NoError(t, err) {
+		assert.Equal(t, 0, len(timeline.Items))
+	}
+}
+
+func TestServer_FollowGetList(t *testing.T) {
+	server, c := createServerClient()
+	defer server.Close()
+	feeds, err := c.FollowGetList("0001")
+	if assert.NoError(t, err) {
+		assert.Equal(t, 1, len(feeds))
+		assert.Equal(t, "test", feeds[0].Name)
+		assert.Equal(t, "feed", feeds[0].Type)
+		assert.Equal(t, "https://example.com/", feeds[0].URL)
+	}
+}
+
+func TestServer_FollowURL(t *testing.T) {
+	server, c := createServerClient()
+	defer server.Close()
+	feed, err := c.FollowURL("0001", "https://example.com/")
+	if assert.NoError(t, err) {
+		assert.Equal(t, "feed", feed.Type)
+		assert.Equal(t, "https://example.com/", feed.URL)
+	}
+}
+
+func TestServer_UnFollowURL(t *testing.T) {
+	server, c := createServerClient()
+	defer server.Close()
+	err := c.UnfollowURL("0001", "https://example.com/")
+	assert.NoError(t, err)
+}
+
+func TestServer_PreviewURL(t *testing.T) {
+	server, c := createServerClient()
+	defer server.Close()
+	timeline, err := c.PreviewURL("https://example.com/")
+	if assert.NoError(t, err) {
+		assert.Equal(t, 0, len(timeline.Items))
+	}
+}
+
+func TestServer_Search(t *testing.T) {
+	server, c := createServerClient()
+	defer server.Close()
+	feeds, err := c.Search("https://example.com/")
+	if assert.NoError(t, err) {
+		assert.Equal(t, 1, len(feeds))
+		assert.Equal(t, "feed", feeds[0].Type)
+		assert.Equal(t, "https://example.com/", feeds[0].URL)
+		assert.Equal(t, "Example", feeds[0].Name)
+		assert.Equal(t, "test.jpg", feeds[0].Photo)
+		assert.Equal(t, "test", feeds[0].Description)
+	}
+}
+
+func TestServer_MarkRead(t *testing.T) {
+	server, c := createServerClient()
+	defer server.Close()
+	err := c.MarkRead("0001", []string{"test"})
+	assert.NoError(t, err)
+}
+
+func TestServer_GetUnknownAction(t *testing.T) {
+	server, c := createServerClient()
+	defer server.Close()
+
+	u := c.MicrosubEndpoint
+	q := u.Query()
+	q.Add("action", "missing")
+	u.RawQuery = q.Encode()
+
+	resp, err := http.Get(u.String())
+	if assert.NoError(t, err) {
+		assert.Equal(t, 400, resp.StatusCode)
+	}
+}
+func TestServer_PostUnknownAction(t *testing.T) {
+	server, c := createServerClient()
+	defer server.Close()
+
+	u := c.MicrosubEndpoint
+	q := u.Query()
+	q.Add("action", "missing")
+	u.RawQuery = q.Encode()
+
+	resp, err := http.Post(u.String(), "application/json", nil)
+	if assert.NoError(t, err) {
+		assert.Equal(t, 400, resp.StatusCode)
+	}
+}

pkg/server/null.go

@@ -15,7 +15,7 @@
    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
-package main
+package server
 
 import (
 	"p83.nl/go/ekster/pkg/microsub"
@@ -25,11 +25,15 @@ import (
 type NullBackend struct {
 }
 
+func (b *NullBackend) AddEventListener(el microsub.EventListener) error {
+	panic("implement me")
+}
+
 // ChannelsGetList gets no channels
 func (b *NullBackend) ChannelsGetList() ([]microsub.Channel, error) {
 	return []microsub.Channel{
-		microsub.Channel{UID: "0001", Name: "notifications", Unread: 0},
+		{UID: "0001", Name: "notifications", Unread: 0},
-		microsub.Channel{UID: "0000", Name: "default", Unread: 0},
+		{UID: "0000", Name: "default", Unread: 0},
 	}, nil
 }
 
@@ -63,7 +67,9 @@ func (b *NullBackend) TimelineGet(before, after, channel string) (microsub.Timel
 }
 
 func (b *NullBackend) FollowGetList(uid string) ([]microsub.Feed, error) {
-	return []microsub.Feed{}, nil
+	return []microsub.Feed{
+		{Name: "test", Type: "feed", URL: "https://example.com/"},
+	}, nil
 }
 
 func (b *NullBackend) FollowURL(uid string, url string) (microsub.Feed, error) {
@@ -75,7 +81,9 @@ func (b *NullBackend) UnfollowURL(uid string, url string) error {
 }
 
 func (b *NullBackend) Search(query string) ([]microsub.Feed, error) {
-	return []microsub.Feed{}, nil
+	return []microsub.Feed{
+		{"feed", "https://example.com/", "Example", "test.jpg", "test", microsub.Card{}},
+	}, nil
 }
 
 func (b *NullBackend) PreviewURL(url string) (microsub.Timeline, error) {