Problem: unread channels are at random positions

Solution: move unread channels to the top

.drone.yml

@@ -1,25 +1,74 @@
+---
 kind: pipeline
-name: default
+type: docker
+name: build and test
 
 workspace:
   base: /go
   path: src/p83.nl/go/ekster
 
+trigger:
+  event:
+    - push
+
+services:
+  - name: redis
+    image: redis:5
+  - name: database
+    image: postgres:14
+    environment:
+      POSTGRES_DB: ekster_testing
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: simple
+      POSTGRES_HOST_AUTH_METHOD: trust
+
 steps:
   - name: testing
-    image: golang:1.16-alpine
+    image: golang:1.18-alpine
     environment:
       CGO_ENABLED: 0
       GOOS: linux
       GOARCH: amd64
     commands:
+      - go version
       - apk --no-cache add git
       - go get -d -t ./...
-      - go build p83.nl/go/ekster/cmd/eksterd
-      - go test ./...
+      - go build -buildvcs=false p83.nl/go/ekster/cmd/eksterd
+      - go vet ./...
+      - go test -v ./...
+
+---
+kind: pipeline
+type: docker
+name: move to production
+
+workspace:
+  base: /go
+  path: src/p83.nl/go/ekster
+
+trigger:
+  event:
+    - promote
+  target:
+    - production
+
+steps:
+  - name: build
+    image: golang:1.18-alpine
+    environment:
+      CGO_ENABLED: 0
+      GOOS: linux
+      GOARCH: amd64
+    commands:
+      - go version
+      - apk --no-cache add git
+      - go get -d -t ./...
+      - go build -buildvcs=false p83.nl/go/ekster/cmd/eksterd
 
   - name: publish-personal
     image: plugins/docker
+    depends_on:
+      - build
     settings:
       repo: registry.stuifzandapp.com/microsub-server
       registry: registry.stuifzandapp.com
@@ -28,19 +77,10 @@ steps:
       password:
         from_secret: docker_password
 
-  - name: publish-docker
-    image: plugins/docker
-    settings:
-      repo: pstuifzand/ekster
-      tags:
-        - alpine
-      username:
-        from_secret: docker_official_username
-      password:
-        from_secret: docker_official_password
-
   - name: deploy
     image: appleboy/drone-ssh
+    depends_on:
+      - publish-personal
     settings:
       host: microsub.stuifzandapp.com
       username: microsub

.github/workflows/go.yml

@@ -0,0 +1,25 @@
+name: Go
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.16
+
+    - name: Build
+      run: go build -v ./...
+
+    - name: Test
+      run: go test -v ./...

.pre-commit-config.yaml

@@ -7,6 +7,7 @@ repos:
     - id: trailing-whitespace
     - id: end-of-file-fixer
     - id: check-yaml
+      args: [--allow-multiple-documents]
     - id: check-merge-conflict
     - id: check-added-large-files
 - repo: https://github.com/dnephin/pre-commit-golang

CHANGELOG.md

@@ -0,0 +1,26 @@
+# Changelog
+
+## [Unreleased]
+
+## [1.0.0-rc.1] - 2021-11-20
+
+### Added
+
+- Postgresql support for channels, feeds, items and subscriptions.
+- Support "source" in items and feeds
+
+### Changed
+
+- Default channel backend is postgresql
+
+### Fixed
+
+- All `staticcheck` problems are fixed.
+- Dependency between memorybackend and hubbackend removed and simplified.
+
+### Deprecated
+
+- All Redis timeline types are deprecated and will be removed in a later version.
+
+[Unreleased]: https://git.p83.nl/peter/ekster/compare/1.0.0-rc.1...master
+[1.0.0-rc.1]: https://git.p83.nl/peter/ekster/src/tag/1.0.0-rc.1

Dockerfile

@@ -3,5 +3,4 @@ RUN apk --no-cache add ca-certificates
 WORKDIR /opt/micropub
 EXPOSE 80
 COPY ./eksterd /app/
-COPY ./templates /app/templates
 ENTRYPOINT ["/app/eksterd"]

README.md

@@ -140,6 +140,8 @@ support microsub.
       -verbose
             show verbose logging
 
+Instead of the `UID` you can also use the `NAME` of the channel in most functions.
+
 ## Configuration: backend.json
 
 The `backend.json` file contains all information about channels, feeds and authentication.
@@ -153,7 +155,6 @@ The two parts that should be changed are:
     "Me": "...",
     "TokenEndpoint": "...",
 
-
 The `Me` value should be set to the URL you use to sign into Monocle, or
 Micropub client.
 

cmd/ek/main.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 // Ek is a microsub client.
 package main
 
@@ -126,7 +144,7 @@ Usage:
 
 Commands:
 
-	connect URL                  login to Indieauth URL, e.g. your website
+	connect URL                  login to a website that supports Indieauth and Microsub
 
 	channels                     list channels
 	channels NAME                create channel with NAME
@@ -218,6 +236,26 @@ Global arguments:
 	performCommands(&c, flag.Args())
 }
 
+func channelID(sub microsub.Microsub, channelNameOrID string) (string, error) {
+	channels, err := sub.ChannelsGetList()
+	if err != nil {
+		// we encountered an error, so we are not sure if it worked
+		return channelNameOrID, err
+	}
+
+	for _, c := range channels {
+		if c.Name == channelNameOrID {
+			return c.UID, nil
+		}
+		if c.UID == channelNameOrID {
+			return c.UID, nil
+		}
+	}
+
+	// unknown?
+	return channelNameOrID, nil
+}
+
 func performCommands(sub microsub.Microsub, commands []string) {
 	if len(commands) == 0 {
 		flag.Usage()
@@ -245,15 +283,15 @@ func performCommands(sub microsub.Microsub, commands []string) {
 	}
 
 	if len(commands) == 3 && commands[0] == "channels" {
-		uid := commands[1]
-		if uid == "-delete" {
-			uid = commands[2]
+		if commands[1] == "-delete" {
+			uid, _ := channelID(sub, commands[2])
 			err := sub.ChannelsDelete(uid)
 			if err != nil {
 				log.Fatalf("An error occurred: %s\n", err)
 			}
 			fmt.Printf("Channel %s deleted\n", uid)
 		} else {
+			uid, _ := channelID(sub, commands[1])
 			name := commands[2]
 			channel, err := sub.ChannelsUpdate(uid, name)
 			if err != nil {
@@ -264,7 +302,7 @@ func performCommands(sub microsub.Microsub, commands []string) {
 	}
 
 	if len(commands) >= 2 && commands[0] == "timeline" {
-		channel := commands[1]
+		channel, _ := channelID(sub, commands[1])
 
 		var timeline microsub.Timeline
 		var err error
@@ -304,7 +342,7 @@ func performCommands(sub microsub.Microsub, commands []string) {
 		query := commands[1]
 		var channel string
 		if len(commands) == 3 {
-			channel = commands[2]
+			channel, _ = channelID(sub, commands[2])
 		} else {
 			channel = "global"
 		}
@@ -331,7 +369,7 @@ func performCommands(sub microsub.Microsub, commands []string) {
 	}
 
 	if len(commands) == 2 && commands[0] == "follow" {
-		uid := commands[1]
+		uid, _ := channelID(sub, commands[1])
 		feeds, err := sub.FollowGetList(uid)
 		if err != nil {
 			log.Fatalf("An error occurred: %s\n", err)
@@ -342,7 +380,7 @@ func performCommands(sub microsub.Microsub, commands []string) {
 	}
 
 	if len(commands) == 3 && commands[0] == "follow" {
-		uid := commands[1]
+		uid, _ := channelID(sub, commands[1])
 		u := commands[2]
 		_, err := sub.FollowURL(uid, u)
 		if err != nil {
@@ -352,7 +390,7 @@ func performCommands(sub microsub.Microsub, commands []string) {
 	}
 
 	if len(commands) == 3 && commands[0] == "unfollow" {
-		uid := commands[1]
+		uid, _ := channelID(sub, commands[1])
 		u := commands[2]
 		err := sub.UnfollowURL(uid, u)
 		if err != nil {

cmd/eksterd/app.go

@@ -0,0 +1,109 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package main
+
+import (
+	"fmt"
+	"log"
+	"math/rand"
+	"net/http"
+	"time"
+
+	"github.com/pkg/errors"
+	"p83.nl/go/ekster/pkg/server"
+)
+
+func init() {
+	rand.Seed(time.Now().UnixNano())
+}
+
+// App is the main app structure
+type App struct {
+	options    AppOptions
+	backend    *memoryBackend
+	hubBackend *hubIncomingBackend
+}
+
+// Run runs the app
+func (app *App) Run() error {
+	err := initSearch()
+	if err != nil {
+		return fmt.Errorf("while starting app: %v", err)
+	}
+	app.backend.run()
+	app.hubBackend.run()
+
+	log.Printf("Listening on port %d\n", app.options.Port)
+	return http.ListenAndServe(fmt.Sprintf(":%d", app.options.Port), nil)
+}
+
+// NewApp initializes the App
+func NewApp(options AppOptions) (*App, error) {
+	app := &App{
+		options: options,
+	}
+
+	backend, err := loadMemoryBackend(options.pool, options.database)
+	if err != nil {
+		return nil, err
+	}
+	app.backend = backend
+
+	// FIXME: load from database
+	app.backend.TokenEndpoint = "https://p83.nl/authtoken"
+	app.backend.Me = "https://p83.nl/"
+
+	app.backend.AuthEnabled = options.AuthEnabled
+
+	app.hubBackend = &hubIncomingBackend{
+		baseURL:  options.BaseURL,
+		pool:     options.pool,
+		database: options.database,
+	}
+	app.backend.hubBackend = app.hubBackend
+
+	http.Handle("/micropub", &micropubHandler{
+		Backend: app.backend,
+		pool:    options.pool,
+	})
+
+	handler, broker := server.NewMicrosubHandler(app.backend)
+	if options.AuthEnabled {
+		handler = WithAuth(handler, app.backend)
+	}
+
+	app.backend.broker = broker
+
+	http.Handle("/microsub", handler)
+
+	http.Handle("/incoming/", &incomingHandler{
+		Backend:   app.hubBackend,
+		Processor: app.backend,
+	})
+
+	if !options.Headless {
+		handler, err := newMainHandler(app.backend, options.BaseURL, options.TemplateDir, options.pool)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not create main handler")
+		}
+		http.Handle("/", handler)
+	}
+
+	return app, nil
+}

cmd/eksterd/auth.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package main
 
 import (

cmd/eksterd/database_test.go

@@ -0,0 +1,123 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2022 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package main
+
+import (
+	"database/sql"
+	"log"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/gomodule/redigo/redis"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+)
+
+type DatabaseSuite struct {
+	suite.Suite
+
+	URL      string
+	Database *sql.DB
+
+	RedisURL string
+	Redis    redis.Conn
+}
+
+func (s *DatabaseSuite) SetupSuite() {
+	db, err := sql.Open("postgres", s.URL)
+	if err != nil {
+		log.Fatal(err)
+	}
+	s.Database = db
+
+	conn, err := redis.Dial("tcp", s.RedisURL)
+	if err != nil {
+		log.Fatal(err)
+	}
+	s.Redis = conn
+	_, err = s.Redis.Do("SELECT", "1")
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func (s *DatabaseSuite) TearDownSuite() {
+	err := s.Database.Close()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = s.Redis.Close()
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+type databaseSuite struct {
+	DatabaseSuite
+}
+
+func (d *databaseSuite) TestGetChannelFromAuthorization() {
+	_, err := d.Database.Exec(`truncate "sources", "channels", "feeds", "subscriptions","items"`)
+	assert.NoError(d.T(), err, "truncate sources, channels, feeds")
+	row := d.Database.QueryRow(`INSERT INTO "channels" (uid, name, created_at, updated_at) VALUES ('abcdef', 'Channel', now(), now()) RETURNING "id"`)
+	var id int
+	err = row.Scan(&id)
+	assert.NoError(d.T(), err, "insert channel")
+	_, err = d.Database.Exec(`INSERT INTO "sources" (channel_id, auth_code, created_at, updated_at) VALUES ($1, '1234', now(), now())`, id)
+	assert.NoError(d.T(), err, "insert sources")
+
+	// source_id found
+	r := httptest.NewRequest("POST", "/micropub?source_id=1234", nil)
+	_, c, err := getChannelFromAuthorization(r, d.Redis, d.Database)
+	assert.NoError(d.T(), err, "channel from source_id")
+	assert.Equal(d.T(), "abcdef", c, "channel uid found")
+
+	// source_id not found
+	r = httptest.NewRequest("POST", "/micropub?source_id=1111", nil)
+	_, c, err = getChannelFromAuthorization(r, d.Redis, d.Database)
+	assert.Error(d.T(), err, "channel from authorization header")
+	assert.Equal(d.T(), "", c, "channel uid found")
+}
+
+func TestDatabaseSuite(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skip test for database")
+	}
+
+	databaseURL := os.Getenv("DATABASE_TEST_URL")
+	if databaseURL == "" {
+		databaseURL = "host=database user=postgres password=simple dbname=ekster_testing sslmode=disable"
+	}
+	databaseSuite := &databaseSuite{
+		DatabaseSuite{
+			URL:      databaseURL,
+			RedisURL: "redis:6379",
+		},
+	}
+
+	databaseURL = "postgres://postgres@database/ekster_testing?sslmode=disable&user=postgres&password=simple"
+	err := runMigrations(databaseURL)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	suite.Run(t, databaseSuite)
+}

cmd/eksterd/db/migrations/000001_create_channels_table.down.sql

@@ -0,0 +1,19 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+DROP TABLE "channels";

cmd/eksterd/db/migrations/000001_create_channels_table.up.sql

@@ -0,0 +1,25 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+CREATE TABLE IF NOT EXISTS "channels" (
+    "id" int primary key generated always as identity,
+    "uid" varchar(255) unique,
+    "name" varchar(255) unique,
+    "created_at" timestamptz DEFAULT current_timestamp,
+    "updated_at" timestamptz
+);

cmd/eksterd/db/migrations/000002_create_feeds_table.down.sql

@@ -0,0 +1,19 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+DROP TABLE "feeds";

cmd/eksterd/db/migrations/000002_create_feeds_table.up.sql

@@ -0,0 +1,25 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+CREATE TABLE "feeds" (
+     "id" int primary key generated always as identity,
+     "channel_id" int references "channels" (id) on update cascade on delete cascade,
+     "url" varchar(512) not null unique,
+     "created_at" timestamptz DEFAULT current_timestamp,
+     "updated_at" timestamptz
+);

cmd/eksterd/db/migrations/000003_add_default_channels.down.sql

@@ -0,0 +1,19 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+DELETE FROM "channels" WHERE "uid" IN ('home', 'notifications');

cmd/eksterd/db/migrations/000003_add_default_channels.up.sql

@@ -0,0 +1,19 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+INSERT INTO "channels" ("uid", "name") VALUES ('home', 'Home'), ('notifications', 'Notifications');

cmd/eksterd/db/migrations/000004_create_items_table.down.sql

@@ -0,0 +1,19 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+DROP TABLE "items";

cmd/eksterd/db/migrations/000004_create_items_table.up.sql

@@ -0,0 +1,28 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+CREATE TABLE IF NOT EXISTS "items" (
+    "id" int primary key generated always as identity,
+    "channel_id" int references "channels" on delete cascade,
+    "uid" varchar(512) not null unique,
+    "is_read" int default 0,
+    "data" jsonb,
+    "created_at" timestamptz DEFAULT current_timestamp,
+    "updated_at" timestamptz,
+    "published_at" timestamptz
+);

cmd/eksterd/db/migrations/000005_alter_items_add_feed_id.down.sql

@@ -0,0 +1,19 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+ALTER TABLE "items" DROP COLUMN "feed_id";

cmd/eksterd/db/migrations/000005_alter_items_add_feed_id.up.sql

@@ -0,0 +1,19 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+ALTER TABLE "items" ADD COLUMN "feed_id" INT REFERENCES "feeds" ON DELETE CASCADE;

cmd/eksterd/db/migrations/000006_create_subscriptions_table.down.sql

@@ -0,0 +1,19 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+DROP TABLE "subscriptions";

cmd/eksterd/db/migrations/000006_create_subscriptions_table.up.sql

@@ -0,0 +1,30 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+CREATE TABLE "subscriptions" (
+     "id" int primary key generated always as identity,
+     "topic" varchar(1024) not null references "feeds" ("url") on update cascade on delete cascade,
+     "hub" varchar(1024) null,
+     "callback" varchar(1024) null,
+     "subscription_secret" varchar(32) not null,
+     "url_secret" varchar(32) not null,
+     "lease_seconds" int not null,
+     "created_at" timestamptz DEFAULT current_timestamp,
+     "updated_at" timestamptz,
+     "resubscribe_at" timestamptz
+);

cmd/eksterd/db/migrations/000007_alter_table_add_feed_tier.down.sql

@@ -0,0 +1,4 @@
+alter table "feeds"
+    drop column "tier",
+    drop column "unmodified",
+    drop column "next_fetch_at";

cmd/eksterd/db/migrations/000007_alter_table_add_feed_tier.up.sql

@@ -0,0 +1,4 @@
+alter table "feeds"
+    add column "tier" int default 0,
+    add column "unmodified" int default 0,
+    add column "next_fetch_at" timestamptz;

cmd/eksterd/db/migrations/000008_create_table_sources.down.sql

@@ -0,0 +1 @@
+DROP TABLE "sources";

cmd/eksterd/db/migrations/000008_create_table_sources.up.sql

@@ -0,0 +1,20 @@
+BEGIN;
+CREATE OR REPLACE FUNCTION update_timestamp()
+    RETURNS TRIGGER AS $$
+BEGIN
+    NEW.updated_at = now();
+    RETURN NEW;
+END;
+$$ language 'plpgsql';
+COMMIT;
+
+CREATE TABLE "sources" (
+   "id" int primary key generated always as identity,
+   "channel_id" int not null,
+   "auth_code" varchar(64) not null,
+   "created_at" timestamp DEFAULT current_timestamp,
+   "updated_at" timestamp DEFAULT current_timestamp
+);
+
+CREATE TRIGGER sources_update_timestamp BEFORE INSERT OR UPDATE ON "sources"
+FOR EACH ROW EXECUTE PROCEDURE update_timestamp();

cmd/eksterd/feedsearch.go

@@ -1,124 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"log"
-	"net/http"
-	"net/url"
-	"strings"
-
-	"p83.nl/go/ekster/pkg/fetch"
-	"p83.nl/go/ekster/pkg/microsub"
-	"willnorris.com/go/microformats"
-)
-
-func isSupportedFeedType(feedType string) bool {
-	return strings.HasPrefix(feedType, "text/html") ||
-		strings.HasPrefix(feedType, "application/json") ||
-		strings.HasPrefix(feedType, "application/xml") ||
-		strings.HasPrefix(feedType, "text/xml") ||
-		strings.HasPrefix(feedType, "application/rss+xml") ||
-		strings.HasPrefix(feedType, "application/atom+xml")
-}
-
-func findFeeds(cachingFetch fetch.FetcherFunc, feedURL string) ([]microsub.Feed, error) {
-	resp, err := cachingFetch(feedURL)
-	if err != nil {
-		return nil, fmt.Errorf("while fetching %s: %w", feedURL, err)
-	}
-	defer resp.Body.Close()
-
-	fetchURL, err := url.Parse(feedURL)
-	md := microformats.Parse(resp.Body, fetchURL)
-	if err != nil {
-		return nil, fmt.Errorf("while fetching %s: %w", feedURL, err)
-	}
-
-	feedResp, err := cachingFetch(fetchURL.String())
-	if err != nil {
-		return nil, fmt.Errorf("in fetch of %s: %w", fetchURL, err)
-	}
-	defer feedResp.Body.Close()
-
-	// TODO: Combine FeedHeader and FeedItems so we can use it here
-	parsedFeed, err := fetch.FeedHeader(cachingFetch, fetchURL.String(), feedResp.Header.Get("Content-Type"), feedResp.Body)
-	if err != nil {
-		return nil, fmt.Errorf("in parse of %s: %w", fetchURL, err)
-	}
-
-	var feeds []microsub.Feed
-
-	// TODO: Only include the feed if it contains some items
-	feeds = append(feeds, parsedFeed)
-
-	// Fetch alternates
-	if alts, e := md.Rels["alternate"]; e {
-		for _, alt := range alts {
-			relURL := md.RelURLs[alt]
-			log.Printf("alternate found with type %s %#v\n", relURL.Type, relURL)
-
-			if isSupportedFeedType(relURL.Type) {
-				parsedFeed, err := fetchAlternateFeed(cachingFetch, alt)
-
-				if err != nil {
-					continue
-				}
-
-				feeds = append(feeds, parsedFeed)
-			}
-		}
-	}
-	return feeds, nil
-}
-
-func fetchAlternateFeed(cachingFetch fetch.FetcherFunc, altURL string) (microsub.Feed, error) {
-	feedResp, err := cachingFetch(altURL)
-	if err != nil {
-		return microsub.Feed{}, fmt.Errorf("fetch of %s: %v", altURL, err)
-	}
-
-	defer feedResp.Body.Close()
-
-	parsedFeed, err := fetch.FeedHeader(cachingFetch, altURL, feedResp.Header.Get("Content-Type"), feedResp.Body)
-	if err != nil {
-		return microsub.Feed{}, fmt.Errorf("in parse of %s: %v", altURL, err)
-	}
-
-	return parsedFeed, nil
-}
-
-func getPossibleURLs(query string) []string {
-	urls := []string{}
-	if !(strings.HasPrefix(query, "https://") || strings.HasPrefix(query, "http://")) {
-		secureURL := "https://" + query
-		if checkURL(secureURL) {
-			urls = append(urls, secureURL)
-		} else {
-			unsecureURL := "http://" + query
-			if checkURL(unsecureURL) {
-				urls = append(urls, unsecureURL)
-			}
-		}
-	} else {
-		urls = append(urls, query)
-	}
-	return urls
-}
-
-func checkURL(u string) bool {
-	testURL, err := url.Parse(u)
-	if err != nil {
-		return false
-	}
-
-	resp, err := http.Head(testURL.String())
-
-	if err != nil {
-		log.Printf("Error while HEAD %s: %v\n", u, err)
-		return false
-	}
-
-	defer resp.Body.Close()
-
-	return resp.StatusCode == 200
-}

cmd/eksterd/http.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package main
 
 import (
@@ -106,10 +124,6 @@ func newMainHandler(backend *memoryBackend, baseURL, templateDir string, pool *r
 	return h, nil
 }
 
-func (h *mainHandler) templateFile(filename string) string {
-	return fmt.Sprintf("%s/%s", h.TemplateDir, filename)
-}
-
 func (h *mainHandler) renderTemplate(w io.Writer, filename string, data interface{}) error {
 	fsys, err := fs.Sub(templates, "templates")
 	if err != nil {
@@ -292,7 +306,7 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	err := r.ParseForm()
 	if err != nil {
 		log.Println(err)
-		http.Error(w, fmt.Sprintf("Bad Request: %s", err.Error()), 400)
+		http.Error(w, fmt.Sprintf("Bad Request: %s", err.Error()), http.StatusBadRequest)
 		return
 	}
 
@@ -317,15 +331,19 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		} else if r.URL.Path == "/session/callback" {
 			c, err := r.Cookie("session")
 			if err == http.ErrNoCookie {
-				http.Redirect(w, r, "/", 302)
+				http.Redirect(w, r, "/", http.StatusFound)
 				return
 			} else if err != nil {
-				http.Error(w, "could not read cookie", 500)
+				http.Error(w, "could not read cookie", http.StatusInternalServerError)
 				return
 			}
 
 			sessionVar := c.Value
 			sess, err := loadSession(sessionVar, conn)
+			if err != nil {
+				fmt.Fprintf(w, "ERROR: %q\n", err)
+				return
+			}
 
 			verified, authResponse, err := performIndieauthCallback(h.BaseURL, r, &sess)
 			if err != nil {
@@ -338,9 +356,9 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 				saveSession(sessionVar, &sess, conn)
 				log.Printf("SESSION: %#v\n", sess)
 				if sess.NextURI != "" {
-					http.Redirect(w, r, sess.NextURI, 302)
+					http.Redirect(w, r, sess.NextURI, http.StatusFound)
 				} else {
-					http.Redirect(w, r, "/", 302)
+					http.Redirect(w, r, "/", http.StatusFound)
 				}
 				return
 			}
@@ -348,11 +366,16 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		} else if r.URL.Path == "/settings/channel" {
 			c, err := r.Cookie("session")
 			if err == http.ErrNoCookie {
-				http.Redirect(w, r, "/", 302)
+				http.Redirect(w, r, "/", http.StatusFound)
 				return
 			}
 			sessionVar := c.Value
 			sess, err := loadSession(sessionVar, conn)
+			if err != nil {
+				log.Printf("ERROR: %s\n", err)
+				http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+				return
+			}
 
 			if !isLoggedIn(h.Backend, &sess) {
 				w.WriteHeader(401)
@@ -362,66 +385,68 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 			var page settingsPage
 			page.Session = sess
-			currentChannelUID := r.URL.Query().Get("uid")
+			currentChannel := r.URL.Query().Get("uid")
 			page.Channels, err = h.Backend.ChannelsGetList()
-			page.Feeds, err = h.Backend.FollowGetList(currentChannelUID)
-
-			var selectedChannel microsub.Channel
-			found := false
-
-			for _, v := range page.Channels {
-				if v.UID == currentChannelUID {
-					selectedChannel = v
-					found = true
-					break
-				}
+			if err != nil {
+				log.Printf("ERROR: %s\n", err)
+				http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+				return
+			}
+			page.Feeds, err = h.Backend.FollowGetList(currentChannel)
+			if err != nil {
+				log.Printf("ERROR: %s\n", err)
+				http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+				return
 			}
 
-			if found {
-				page.CurrentChannel = selectedChannel
-				if setting, e := h.Backend.Settings[selectedChannel.UID]; e {
-					page.CurrentSetting = setting
-				} else {
-					page.CurrentSetting = channelSetting{}
-				}
-				// FIXME: similar code is found in timeline.go
-				if page.CurrentSetting.ChannelType == "" {
-					if selectedChannel.UID == "notifications" {
-						page.CurrentSetting.ChannelType = "stream"
+			for _, v := range page.Channels {
+				if v.UID == currentChannel {
+					page.CurrentChannel = v
+					if setting, e := h.Backend.Settings[v.UID]; e {
+						page.CurrentSetting = setting
 					} else {
-						page.CurrentSetting.ChannelType = "sorted-set"
+						page.CurrentSetting = channelSetting{}
 					}
-				}
-				page.ExcludedTypeNames = map[string]string{
-					"repost":   "Reposts",
-					"like":     "Likes",
-					"bookmark": "Bookmarks",
-					"reply":    "Replies",
-					"checkin":  "Checkins",
-				}
-				page.ExcludedTypes = make(map[string]bool)
-				types := []string{"repost", "like", "bookmark", "reply", "checkin"}
-				for _, v := range types {
-					page.ExcludedTypes[v] = false
-				}
-				for _, v := range page.CurrentSetting.ExcludeType {
-					page.ExcludedTypes[v] = true
+					if page.CurrentSetting.ChannelType == "" {
+						page.CurrentSetting.ChannelType = "postgres-stream"
+					}
+					page.ExcludedTypeNames = map[string]string{
+						"repost":   "Reposts",
+						"like":     "Likes",
+						"bookmark": "Bookmarks",
+						"reply":    "Replies",
+						"checkin":  "Checkins",
+					}
+					page.ExcludedTypes = make(map[string]bool)
+					types := []string{"repost", "like", "bookmark", "reply", "checkin"}
+					for _, v := range types {
+						page.ExcludedTypes[v] = false
+					}
+					for _, v := range page.CurrentSetting.ExcludeType {
+						page.ExcludedTypes[v] = true
+					}
+					break
 				}
 			}
 
 			err = h.renderTemplate(w, "channel.html", page)
 			if err != nil {
-				http.Error(w, err.Error(), 500)
+				fmt.Fprintf(w, "ERROR: %s\n", err)
 			}
 			return
 		} else if r.URL.Path == "/logs" {
 			c, err := r.Cookie("session")
 			if err == http.ErrNoCookie {
-				http.Redirect(w, r, "/", 302)
+				http.Redirect(w, r, "/", http.StatusFound)
 				return
 			}
 			sessionVar := c.Value
 			sess, err := loadSession(sessionVar, conn)
+			if err != nil {
+				log.Printf("ERROR: %s\n", err)
+				http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+				return
+			}
 
 			if !isLoggedIn(h.Backend, &sess) {
 				w.WriteHeader(401)
@@ -440,11 +465,16 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		} else if r.URL.Path == "/settings" {
 			c, err := r.Cookie("session")
 			if err == http.ErrNoCookie {
-				http.Redirect(w, r, "/", 302)
+				http.Redirect(w, r, "/", http.StatusFound)
 				return
 			}
 			sessionVar := c.Value
 			sess, err := loadSession(sessionVar, conn)
+			if err != nil {
+				log.Printf("ERROR: %s\n", err)
+				http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+				return
+			}
 
 			if !isLoggedIn(h.Backend, &sess) {
 				w.WriteHeader(401)
@@ -455,6 +485,11 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			var page settingsPage
 			page.Session = sess
 			page.Channels, err = h.Backend.ChannelsGetList()
+			if err != nil {
+				log.Printf("ERROR: %s\n", err)
+				http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+				return
+			}
 			// page.Feeds = h.Backend.Feeds
 
 			err = h.renderTemplate(w, "settings.html", page)
@@ -469,11 +504,16 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			sessionVar := getSessionCookie(w, r)
 
 			sess, err := loadSession(sessionVar, conn)
+			if err != nil {
+				log.Printf("ERROR: %s\n", err)
+				http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+				return
+			}
 
 			if !isLoggedIn(h.Backend, &sess) {
 				sess.NextURI = r.URL.String()
 				saveSession(sessionVar, &sess, conn)
-				http.Redirect(w, r, "/", 302)
+				http.Redirect(w, r, "/", http.StatusFound)
 				return
 			}
 
@@ -515,6 +555,11 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			page.Scope = scope
 			page.State = state
 			page.Channels, err = h.Backend.ChannelsGetList()
+			if err != nil {
+				log.Printf("ERROR: %s\n", err)
+				http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+				return
+			}
 
 			app, err := getAppInfo(clientID)
 			if err != nil {
@@ -532,7 +577,7 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/session" {
 			c, err := r.Cookie("session")
 			if err == http.ErrNoCookie {
-				http.Redirect(w, r, "/", 302)
+				http.Redirect(w, r, "/", http.StatusFound)
 				return
 			}
 
@@ -543,7 +588,7 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 			endpoints, err := getEndpoints(me)
 			if err != nil {
-				http.Error(w, fmt.Sprintf("Bad Request: %s, %s", err.Error(), me), 400)
+				http.Error(w, fmt.Sprintf("Bad Request: %s, %s", err.Error(), me), http.StatusBadRequest)
 				return
 			}
 
@@ -553,7 +598,7 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			sess, err := loadSession(sessionVar, conn)
 
 			if err != nil {
-				http.Redirect(w, r, "/", 302)
+				http.Redirect(w, r, "/", http.StatusFound)
 				return
 			}
 
@@ -565,12 +610,12 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 			err = saveSession(sessionVar, &sess, conn)
 			if err != nil {
-				http.Redirect(w, r, "/", 302)
+				http.Redirect(w, r, "/", http.StatusFound)
 				return
 			}
 
 			authenticationURL := indieauth.CreateAuthenticationURL(*endpoints.AuthorizationEndpoint, endpoints.Me.String(), h.BaseURL, redirectURI, state)
-			http.Redirect(w, r, authenticationURL, 302)
+			http.Redirect(w, r, authenticationURL, http.StatusFound)
 
 			return
 		} else if r.URL.Path == "/session/logout" {
@@ -623,7 +668,7 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			redirectURI.RawQuery = q.Encode()
 
 			log.Println(redirectURI)
-			http.Redirect(w, r, redirectURI.String(), 302)
+			http.Redirect(w, r, redirectURI.String(), http.StatusFound)
 			return
 		} else if r.URL.Path == "/auth/token" {
 			grantType := r.FormValue("grant_type")
@@ -666,45 +711,40 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			}
 
 			w.Header().Add("Content-Type", "application/json")
-			enc := json.NewEncoder(w)
-			err = enc.Encode(&res)
-			if err != nil {
+			if err := json.NewEncoder(w).Encode(&res); err != nil {
 				log.Println(err)
-				fmt.Fprintf(w, "ERROR: %q", err)
 				return
 			}
 			return
 		} else if r.URL.Path == "/settings/channel" {
-			defer h.Backend.save()
-			uid := r.FormValue("uid")
+			// defer h.Backend.save()
+			// uid := r.FormValue("uid")
+			//
+			// if h.Backend.Settings == nil {
+			// 	h.Backend.Settings = make(map[string]channelSetting)
+			// }
+			//
+			// excludeRegex := r.FormValue("exclude_regex")
+			// includeRegex := r.FormValue("include_regex")
+			// channelType := r.FormValue("type")
+			//
+			// setting, e := h.Backend.Settings[uid]
+			// if !e {
+			// 	setting = channelSetting{}
+			// }
+			// setting.ExcludeRegex = excludeRegex
+			// setting.IncludeRegex = includeRegex
+			// setting.ChannelType = channelType
+			// if values, e := r.Form["exclude_type"]; e {
+			// 	setting.ExcludeType = values
+			// }
+			// h.Backend.Settings[uid] = setting
 
-			if h.Backend.Settings == nil {
-				h.Backend.Settings = make(map[string]channelSetting)
-			}
-
-			excludeRegex := r.FormValue("exclude_regex")
-			includeRegex := r.FormValue("include_regex")
-			channelType := r.FormValue("type")
-
-			setting, e := h.Backend.Settings[uid]
-			if !e {
-				setting = channelSetting{}
-			}
-			setting.ExcludeRegex = excludeRegex
-			setting.IncludeRegex = includeRegex
-			setting.ChannelType = channelType
-			if values, e := r.Form["exclude_type"]; e {
-				setting.ExcludeType = values
-			}
-			h.Backend.Settings[uid] = setting
-
-			h.Backend.Debug()
-
-			http.Redirect(w, r, "/settings", 302)
+			http.Redirect(w, r, "/settings", http.StatusFound)
 			return
 		} else if r.URL.Path == "/refresh" {
 			h.Backend.RefreshFeeds()
-			http.Redirect(w, r, "/", 302)
+			http.Redirect(w, r, "/", http.StatusFound)
 			return
 		}
 	}
@@ -715,14 +755,14 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 func httpSessionLogout(r *http.Request, w http.ResponseWriter, conn redis.Conn) {
 	c, err := r.Cookie("session")
 	if err == http.ErrNoCookie {
-		http.Redirect(w, r, "/", 302)
+		http.Redirect(w, r, "/", http.StatusFound)
 		return
 	}
 	if err == nil {
 		sessionVar := c.Value
 		_, _ = conn.Do("DEL", "session:"+sessionVar)
 	}
-	http.Redirect(w, r, "/", 302)
+	http.Redirect(w, r, "/", http.StatusFound)
 }
 
 type parsedEndpoints struct {

cmd/eksterd/hubbackend.go

@@ -1,17 +1,32 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package main
 
 import (
+	"database/sql"
 	"expvar"
 	"fmt"
 	"io"
 	"log"
 	"net/http"
-	"net/url"
-	"strconv"
-	"strings"
 	"time"
 
-	"github.com/pkg/errors"
 	"p83.nl/go/ekster/pkg/util"
 	"p83.nl/go/ekster/pkg/websub"
 
@@ -23,31 +38,29 @@ const LeaseSeconds = 24 * 60 * 60
 
 // HubBackend handles information for the incoming handler
 type HubBackend interface {
-	GetFeeds() []Feed // Deprecated
 	Feeds() ([]Feed, error)
-	CreateFeed(url, channel string) (int64, error)
+	CreateFeed(url string) (int64, error)
 	GetSecret(feedID int64) string
-	UpdateFeed(feedID int64, contentType string, body io.Reader) error
+	UpdateFeed(processor ContentProcessor, feedID int64, contentType string, body io.Reader) error
 	FeedSetLeaseSeconds(feedID int64, leaseSeconds int64) error
 	Subscribe(feed *Feed) error
 }
 
 type hubIncomingBackend struct {
-	backend *memoryBackend
-	baseURL string
-	pool    *redis.Pool
+	baseURL  string
+	pool     *redis.Pool
+	database *sql.DB
 }
 
 // Feed contains information about the feed subscriptions
 type Feed struct {
-	ID            int64  `redis:"id"`
-	Channel       string `redis:"channel"`
-	URL           string `redis:"url"`
-	Callback      string `redis:"callback"`
-	Hub           string `redis:"hub"`
-	Secret        string `redis:"secret"`
-	LeaseSeconds  int64  `redis:"lease_seconds"`
-	ResubscribeAt int64  `redis:"resubscribe_at"`
+	ID            int64
+	URL           string
+	Callback      string
+	Hub           string
+	Secret        string
+	LeaseSeconds  int64
+	ResubscribeAt *time.Time
 }
 
 var (
@@ -59,29 +72,35 @@ func init() {
 }
 
 func (h *hubIncomingBackend) GetSecret(id int64) string {
-	conn := h.pool.Get()
-	defer conn.Close()
-	secret, err := redis.String(conn.Do("HGET", fmt.Sprintf("feed:%d", id), "secret"))
+	db := h.database
+	var secret string
+	err := db.QueryRow(
+		`select "subscription_secret" from "subscriptions" where "id" = $1`,
+		id,
+	).Scan(&secret)
 	if err != nil {
 		return ""
 	}
 	return secret
 }
 
-func (h *hubIncomingBackend) CreateFeed(topic string, channel string) (int64, error) {
-	conn := h.pool.Get()
-	defer conn.Close()
+func (h *hubIncomingBackend) CreateFeed(topic string) (int64, error) {
+	log.Println("CreateFeed", topic)
+	db := h.database
 
-	// TODO(peter): check if topic already is registered
-	id, err := redis.Int64(conn.Do("INCR", "feed:next_id"))
+	secret := util.RandStringBytes(32)
+	urlSecret := util.RandStringBytes(32)
+
+	var subscriptionID int
+	err := db.QueryRow(`
+INSERT INTO "subscriptions" ("topic","subscription_secret", "url_secret", "lease_seconds", "created_at")
+VALUES ($1, $2, $3, $4, DEFAULT) RETURNING "id"`, topic, secret, urlSecret, 60*60*24*7).Scan(&subscriptionID)
 	if err != nil {
 		return 0, err
 	}
-
-	conn.Do("HSET", fmt.Sprintf("feed:%d", id), "url", topic)
-	conn.Do("HSET", fmt.Sprintf("feed:%d", id), "channel", channel)
-	secret := util.RandStringBytes(16)
-	conn.Do("HSET", fmt.Sprintf("feed:%d", id), "secret", secret)
+	if err != nil {
+		return 0, fmt.Errorf("insert into subscriptions: %w", err)
+	}
 
 	client := &http.Client{}
 
@@ -91,135 +110,107 @@ func (h *hubIncomingBackend) CreateFeed(topic string, channel string) (int64, er
 		return 0, err
 	}
 
-	callbackURL := fmt.Sprintf("%s/incoming/%d", h.baseURL, id)
+	callbackURL := fmt.Sprintf("%s/incoming/%d", h.baseURL, subscriptionID)
 
 	log.Printf("WebSub Hub URL found for topic=%q hub=%q callback=%q\n", topic, hubURL, callbackURL)
 
 	if err == nil && hubURL != "" {
-		args := redis.Args{}.Add(fmt.Sprintf("feed:%d", id), "hub", hubURL, "callback", callbackURL)
-		_, err = conn.Do("HMSET", args...)
+		_, err := db.Exec(`UPDATE subscriptions SET hub = $1, callback = $2 WHERE id = $3`, hubURL, callbackURL, subscriptionID)
 		if err != nil {
-			return 0, errors.Wrap(err, "could not write to redis backend")
+			return 0, fmt.Errorf("save hub and callback: %w", err)
 		}
 	} else {
-		return id, nil
+		return int64(subscriptionID), nil
 	}
 
 	err = websub.Subscribe(client, hubURL, topic, callbackURL, secret, 24*3600)
 	if err != nil {
-		return 0, err
+		return 0, fmt.Errorf("subscribe: %w", err)
 	}
 
-	return id, nil
+	return int64(subscriptionID), nil
 }
 
-func (h *hubIncomingBackend) UpdateFeed(feedID int64, contentType string, body io.Reader) error {
-	conn := h.pool.Get()
-	defer conn.Close()
-	log.Printf("updating feed %d", feedID)
-	u, err := redis.String(conn.Do("HGET", fmt.Sprintf("feed:%d", feedID), "url"))
-	if err != nil {
-		return err
-	}
-	channel, err := redis.String(conn.Do("HGET", fmt.Sprintf("feed:%d", feedID), "channel"))
+func (h *hubIncomingBackend) UpdateFeed(processor ContentProcessor, subscriptionID int64, contentType string, body io.Reader) error {
+	log.Println("UpdateFeed", subscriptionID)
+
+	db := h.database
+	// Process all channels that contains this feed
+	rows, err := db.Query(`
+select topic, c.uid, f.id, c.name
+from subscriptions s
+inner join feeds f    on f.url = s.topic
+inner join channels c on c.id = f.channel_id
+where s.id = $1
+`,
+		subscriptionID,
+	)
 	if err != nil {
 		return err
 	}
 
-	log.Printf("Updating feed %d - %s %s\n", feedID, u, channel)
-	err = h.backend.ProcessContent(channel, u, contentType, body)
-	if err != nil {
-		log.Printf("could not process content for channel %s: %s", channel, err)
+	for rows.Next() {
+		var topic, channel, feedID, channelName string
+
+		err = rows.Scan(&topic, &channel, &feedID, &channelName)
+		if err != nil {
+			log.Println(err)
+			continue
+		}
+
+		log.Printf("Updating feed %s %q in %q (%s)\n", feedID, topic, channelName, channel)
+		_, err = processor.ProcessContent(channel, feedID, topic, contentType, body)
+		if err != nil {
+			log.Printf("could not process content for channel %s: %s", channelName, err)
+		}
 	}
 
 	return err
 }
 
-func (h *hubIncomingBackend) FeedSetLeaseSeconds(feedID int64, leaseSeconds int64) error {
-	conn := h.pool.Get()
-	defer conn.Close()
-	log.Printf("updating feed %d lease_seconds", feedID)
-
-	args := redis.Args{}.Add(fmt.Sprintf("feed:%d", feedID), "lease_seconds", leaseSeconds, "resubscribe_at", time.Now().Add(time.Duration(60*(leaseSeconds-15))*time.Second).Unix())
-	_, err := conn.Do("HMSET", args...)
-	if err != nil {
-		log.Println(err)
-		return err
-	}
-
-	return nil
-}
-
-// GetFeeds is deprecated, use Feeds instead
-func (h *hubIncomingBackend) GetFeeds() []Feed {
-	log.Println("GetFeeds called, consider replacing with Feeds")
-	feeds, err := h.Feeds()
-	if err != nil {
-		log.Printf("Feeds returned an error: %v", err)
-	}
-	return feeds
+func (h *hubIncomingBackend) FeedSetLeaseSeconds(subscriptionID int64, leaseSeconds int64) error {
+	db := h.database
+	_, err := db.Exec(`
+update subscriptions
+set lease_seconds = $1,
+    resubscribe_at = now() + $2 * interval '1' second
+where id = $3
+`, leaseSeconds, leaseSeconds, subscriptionID)
+	return err
 }
 
 // Feeds returns a list of subscribed feeds
 func (h *hubIncomingBackend) Feeds() ([]Feed, error) {
-	conn := h.pool.Get()
-	defer conn.Close()
-	feeds := []Feed{}
+	db := h.database
+	var feeds []Feed
 
-	// FIXME(peter): replace with set of currently checked feeds
-	feedKeys, err := redis.Strings(conn.Do("KEYS", "feed:*"))
+	rows, err := db.Query(`
+		select s.id, topic, hub, callback, subscription_secret, lease_seconds, resubscribe_at
+		from subscriptions s
+		inner join feeds f on f.url = s.topic
+		inner join channels c on c.id = f.channel_id
+		where hub is not null
+	`)
 	if err != nil {
-		return nil, errors.Wrap(err, "could not get feeds from backend")
+		return nil, err
 	}
 
-	for _, feedKey := range feedKeys {
+	for rows.Next() {
 		var feed Feed
-		values, err := redis.Values(conn.Do("HGETALL", feedKey))
+
+		err = rows.Scan(
+			&feed.ID,
+			&feed.URL,
+			&feed.Hub,
+			&feed.Callback,
+			&feed.Secret,
+			&feed.LeaseSeconds,
+			&feed.ResubscribeAt,
+		)
 		if err != nil {
-			log.Printf("could not get feed info for key %s: %v", feedKey, err)
+			log.Println("Feeds: scan subscriptions:", err)
 			continue
 		}
-
-		err = redis.ScanStruct(values, &feed)
-		if err != nil {
-			log.Printf("could not scan struct for key %s: %v", feedKey, err)
-			continue
-		}
-
-		// Add feed id
-		if feed.ID == 0 {
-			parts := strings.Split(feedKey, ":")
-			if len(parts) == 2 {
-				feed.ID, _ = strconv.ParseInt(parts[1], 10, 64)
-				_, err = conn.Do("HSET", feedKey, "id", feed.ID)
-				if err != nil {
-					log.Printf("could not save id for %s: %v", feedKey, err)
-				}
-			}
-		}
-
-		// Fix the callback url
-		callbackURL, err := url.Parse(feed.Callback)
-		if err != nil || !callbackURL.IsAbs() {
-			if err != nil {
-				log.Printf("could not parse callback url %q: %v", callbackURL, err)
-			} else {
-				log.Printf("url is relative; replace with absolute url: %q", callbackURL)
-			}
-
-			feed.Callback = fmt.Sprintf("%s/incoming/%d", h.baseURL, feed.ID)
-			_, err = conn.Do("HSET", feedKey, "callback", feed.Callback)
-			if err != nil {
-				log.Printf("could not save id for %s: %v", feedKey, err)
-			}
-		}
-
-		// Skip feeds without a Hub
-		if feed.Hub == "" {
-			continue
-		}
-
-		log.Printf("Websub feed: %#v\n", feed)
 		feeds = append(feeds, feed)
 	}
 
@@ -227,28 +218,33 @@ func (h *hubIncomingBackend) Feeds() ([]Feed, error) {
 }
 
 func (h *hubIncomingBackend) Subscribe(feed *Feed) error {
+	log.Println("Subscribe", feed.URL)
 	client := http.Client{}
 	return websub.Subscribe(&client, feed.Hub, feed.URL, feed.Callback, feed.Secret, LeaseSeconds)
 }
 
 func (h *hubIncomingBackend) run() error {
-	ticker := time.NewTicker(10 * time.Minute)
+	ticker := time.NewTicker(1 * time.Minute)
 	quit := make(chan struct{})
 
 	go func() {
 		for {
 			select {
 			case <-ticker.C:
-				log.Println("Getting feeds for WebSub")
+				log.Println("Getting feeds for WebSub started")
 				varWebsub.Add("runs", 1)
 
 				feeds, err := h.Feeds()
 				if err != nil {
+					log.Println("Feeds failed:", err)
+					log.Println("Getting feeds for WebSub completed")
+					continue
 				}
 
+				log.Printf("Found %d feeds", len(feeds))
 				for _, feed := range feeds {
 					log.Printf("Looking at %s\n", feed.URL)
-					if feed.ResubscribeAt == 0 || time.Now().After(time.Unix(feed.ResubscribeAt, 0)) {
+					if feed.ResubscribeAt != nil && time.Now().After(*feed.ResubscribeAt) {
 						if feed.Callback == "" {
 							feed.Callback = fmt.Sprintf("%s/incoming/%d", h.baseURL, feed.ID)
 						}
@@ -261,6 +257,8 @@ func (h *hubIncomingBackend) run() error {
 						}
 					}
 				}
+
+				log.Println("Getting feeds for WebSub completed")
 			case <-quit:
 				ticker.Stop()
 				return

cmd/eksterd/incoming.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package main
 
 import (
@@ -13,7 +31,8 @@ import (
 )
 
 type incomingHandler struct {
-	Backend HubBackend
+	Backend   HubBackend
+	Processor ContentProcessor
 }
 
 var (
@@ -49,12 +68,14 @@ func (h *incomingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 			leaseSeconds, err := strconv.ParseInt(leaseStr, 10, 64)
 			if err != nil {
-				http.Error(w, fmt.Sprintf("error in hub.lease_seconds format %q: %s", leaseSeconds, err), 400)
+				log.Printf("error in hub.lease_seconds format %q: %s", leaseSeconds, err)
+				http.Error(w, fmt.Sprintf("error in hub.lease_seconds format %q: %s", leaseSeconds, err), http.StatusBadRequest)
 				return
 			}
 			err = h.Backend.FeedSetLeaseSeconds(feed, leaseSeconds)
 			if err != nil {
-				http.Error(w, fmt.Sprintf("error in while setting hub.lease_seconds: %s", err), 400)
+				log.Printf("error in while setting hub.lease_seconds: %s", err)
+				http.Error(w, fmt.Sprintf("error in while setting hub.lease_seconds: %s", err), http.StatusBadRequest)
 				return
 			}
 		}
@@ -67,7 +88,7 @@ func (h *incomingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if r.Method != http.MethodPost {
-		http.Error(w, "Method not allowed", 405)
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
 		return
 	}
 
@@ -75,28 +96,31 @@ func (h *incomingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	secret := h.Backend.GetSecret(feed)
 	if secret == "" {
 		log.Printf("missing secret for feed %d\n", feed)
-		http.Error(w, "Unknown", 400)
+		http.Error(w, "Unknown", http.StatusBadRequest)
 		return
 	}
 
 	feedContent, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		log.Printf("ERROR: %s\n", err)
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		return
+	}
 
 	// match signature
 	sig := r.Header.Get("X-Hub-Signature")
 	if sig != "" {
 		if err := websub.ValidateHubSignature(sig, feedContent, []byte(secret)); err != nil {
 			log.Printf("could not validate signature: %+v", err)
-			http.Error(w, fmt.Sprintf("could not validate signature: %s", err), 400)
+			http.Error(w, fmt.Sprintf("could not validate signature: %s", err), http.StatusBadRequest)
 			return
 		}
 	}
 
 	ct := r.Header.Get("Content-Type")
-	err = h.Backend.UpdateFeed(feed, ct, bytes.NewBuffer(feedContent))
+	err = h.Backend.UpdateFeed(h.Processor, feed, ct, bytes.NewBuffer(feedContent))
 	if err != nil {
-		http.Error(w, fmt.Sprintf("could not update feed: %s (%s)", ct, err), 400)
+		http.Error(w, fmt.Sprintf("could not update feed: %s (%s)", ct, err), http.StatusBadRequest)
 		return
 	}
-
-	return
 }

cmd/eksterd/main.go

@@ -1,14 +1,20 @@
-// Copyright (C) 2018 Peter Stuifzand
-//
-// This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public
-// License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any
-// later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
-// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along with this program. If not,
-// see <http://www.gnu.org/licenses/>.
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
 
 /*
 Eksterd is a microsub server that is extendable.
@@ -17,18 +23,20 @@ package main
 
 import (
 	"database/sql"
+	"embed"
+	_ "expvar"
 	"flag"
-	"fmt"
 	"log"
 	"net/http"
 	"os"
 	"time"
 
-	"github.com/gomodule/redigo/redis"
-	"github.com/pkg/errors"
 	"p83.nl/go/ekster/pkg/auth"
 
-	"p83.nl/go/ekster/pkg/server"
+	"github.com/golang-migrate/migrate/v4"
+	_ "github.com/golang-migrate/migrate/v4/database/postgres"
+	"github.com/golang-migrate/migrate/v4/source/iofs"
+	"github.com/gomodule/redigo/redis"
 )
 
 // AppOptions are options for the app
@@ -43,6 +51,9 @@ type AppOptions struct {
 	database    *sql.DB
 }
 
+//go:embed db/migrations/*.sql
+var migrations embed.FS
+
 func init() {
 	log.SetFlags(log.Lshortfile | log.Ldate | log.Ltime)
 }
@@ -81,13 +92,13 @@ func WithAuth(handler http.Handler, b *memoryBackend) http.Handler {
 		}
 		if !authorized {
 			log.Printf("Token could not be validated")
-			http.Error(w, "Can't validate token", 403)
+			http.Error(w, "Can't validate token", http.StatusForbidden)
 			return
 		}
 
 		if token.Me != b.Me { // FIXME: Me should be part of the request
 			log.Printf("Missing \"me\" in token response: %#v\n", token)
-			http.Error(w, "Wrong me", 403)
+			http.Error(w, "Wrong me", http.StatusForbidden)
 			return
 		}
 
@@ -95,73 +106,6 @@ func WithAuth(handler http.Handler, b *memoryBackend) http.Handler {
 	})
 }
 
-// App is the main app structure
-type App struct {
-	options    AppOptions
-	backend    *memoryBackend
-	hubBackend *hubIncomingBackend
-}
-
-// Run runs the app
-func (app *App) Run() error {
-	err := initSearch()
-	if err != nil {
-		return fmt.Errorf("while starting app: %v", err)
-	}
-	app.backend.run()
-	app.hubBackend.run()
-
-	log.Printf("Listening on port %d\n", app.options.Port)
-	return http.ListenAndServe(fmt.Sprintf(":%d", app.options.Port), nil)
-}
-
-// NewApp initializes the App
-func NewApp(options AppOptions) (*App, error) {
-	app := &App{
-		options: options,
-	}
-
-	backend, err := loadMemoryBackend(options.pool, options.database)
-	if err != nil {
-		return nil, err
-	}
-	app.backend = backend
-	app.backend.AuthEnabled = options.AuthEnabled
-	app.backend.baseURL = options.BaseURL
-	app.backend.hubIncomingBackend.pool = options.pool
-	app.backend.hubIncomingBackend.baseURL = options.BaseURL
-
-	app.hubBackend = &hubIncomingBackend{backend: app.backend, baseURL: options.BaseURL, pool: options.pool}
-
-	http.Handle("/micropub", &micropubHandler{
-		Backend: app.backend,
-		pool:    options.pool,
-	})
-
-	handler, broker := server.NewMicrosubHandler(app.backend)
-	if options.AuthEnabled {
-		handler = WithAuth(handler, app.backend)
-	}
-
-	app.backend.broker = broker
-
-	http.Handle("/microsub", handler)
-
-	http.Handle("/incoming/", &incomingHandler{
-		Backend: app.hubBackend,
-	})
-
-	if !options.Headless {
-		handler, err := newMainHandler(app.backend, options.BaseURL, options.TemplateDir, options.pool)
-		if err != nil {
-			return nil, errors.Wrap(err, "could not create main handler")
-		}
-		http.Handle("/", handler)
-	}
-
-	return app, nil
-}
-
 func main() {
 	log.Println("eksterd - microsub server")
 
@@ -197,28 +141,36 @@ func main() {
 			log.Fatal("EKSTER_TEMPLATES environment variable not found, use env var or -templates dir option")
 		}
 	}
+	//
+	// createBackend := false
+	// args := flag.Args()
+	//
+	// if len(args) >= 1 {
+	// 	if args[0] == "new" {
+	// 		createBackend = true
+	// 	}
+	// }
+	//
+	// if createBackend {
+	// 	err := createMemoryBackend()
+	// 	if err != nil {
+	// 		log.Fatalf("Error while saving backend.json: %s", err)
+	// 	}
+	//
+	// TODO(peter): automatically gather this information from login or otherwise
+	//
+	// 	log.Println(`Config file "backend.json" is created in the current directory.`)
+	// 	log.Println(`Update "Me" variable to your website address "https://example.com/"`)
+	// 	log.Println(`Update "TokenEndpoint" variable to the address of your token endpoint "https://example.com/token"`)
+	//
+	// 	return
+	// }
 
-	createBackend := false
-	args := flag.Args()
-
-	if len(args) >= 1 {
-		if args[0] == "new" {
-			createBackend = true
-		}
-	}
-
-	if createBackend {
-		err := createMemoryBackend()
-		if err != nil {
-			log.Fatalf("Error while saving backend.json: %s", err)
-		}
-
-		// TODO(peter): automatically gather this information from login or otherwise
-		log.Println(`Config file "backend.json" is created in the current directory.`)
-		log.Println(`Update "Me" variable to your website address "https://example.com/"`)
-		log.Println(`Update "TokenEndpoint" variable to the address of your token endpoint "https://example.com/token"`)
-
-		return
+	// TODO(peter): automatically gather this information from login or otherwise
+	databaseURL := "postgres://postgres@database/ekster?sslmode=disable&user=postgres&password=simple"
+	err := runMigrations(databaseURL)
+	if err != nil {
+		log.Fatalf("Error with migrations: %s", err)
 	}
 
 	pool := newPool(options.RedisServer)
@@ -238,3 +190,38 @@ func main() {
 
 	db.Close()
 }
+
+// Log migrations
+type Log struct {
+}
+
+// Printf for migrations logs
+func (l Log) Printf(format string, v ...interface{}) {
+	log.Printf(format, v...)
+}
+
+// Verbose returns false
+func (l Log) Verbose() bool {
+	return false
+}
+
+func runMigrations(databaseURL string) error {
+	d, err := iofs.New(migrations, "db/migrations")
+	if err != nil {
+		return err
+	}
+	m, err := migrate.NewWithSourceInstance("iofs", d, databaseURL)
+	if err != nil {
+		return err
+	}
+	defer m.Close()
+	m.Log = &Log{}
+	log.Println("Running migrations")
+	if err = m.Up(); err != nil {
+		if err != migrate.ErrNoChange {
+			return err
+		}
+	}
+	log.Println("Migrations are up")
+	return nil
+}

cmd/eksterd/memory_test.go

@@ -1,114 +1,228 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package main
 
-import (
-	"reflect"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/gomodule/redigo/redis"
-	"p83.nl/go/ekster/pkg/microsub"
-	"p83.nl/go/ekster/pkg/sse"
-)
-
-func Test_memoryBackend_ChannelsCreate(t *testing.T) {
-	type fields struct {
-		hubIncomingBackend hubIncomingBackend
-		lock               sync.RWMutex
-		Channels           map[string]microsub.Channel
-		Feeds              map[string][]microsub.Feed
-		Settings           map[string]channelSetting
-		NextUID            int
-		Me                 string
-		TokenEndpoint      string
-		AuthEnabled        bool
-		ticker             *time.Ticker
-		quit               chan struct{}
-		broker             *sse.Broker
-		pool               *redis.Pool
-	}
-	type args struct {
-		name string
-	}
-	tests := []struct {
-		name    string
-		fields  fields
-		args    args
-		want    microsub.Channel
-		wantErr bool
-	}{
-		{
-			name: "Duplicate channel",
-			fields: fields{
-				hubIncomingBackend: hubIncomingBackend{},
-				lock:               sync.RWMutex{},
-				Channels: func() map[string]microsub.Channel {
-					channels := make(map[string]microsub.Channel)
-					channels["1234"] = microsub.Channel{
-						UID:  "1234",
-						Name: "Test",
-						Unread: microsub.Unread{
-							Type:        microsub.UnreadCount,
-							Unread:      false,
-							UnreadCount: 0,
-						},
-					}
-					return channels
-				}(),
-				Feeds: func() map[string][]microsub.Feed {
-					feeds := make(map[string][]microsub.Feed)
-					return feeds
-				}(),
-				Settings:      nil,
-				NextUID:       1,
-				Me:            "",
-				TokenEndpoint: "",
-				AuthEnabled:   false,
-				ticker:        nil,
-				quit:          nil,
-				broker:        nil,
-				pool:          nil,
-			},
-			args: args{
-				name: "Test",
-			},
-			want: microsub.Channel{
-				UID:  "1234",
-				Name: "Test",
-				Unread: microsub.Unread{
-					Type:        microsub.UnreadCount,
-					Unread:      false,
-					UnreadCount: 0,
-				},
-			},
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			b := &memoryBackend{
-				hubIncomingBackend: tt.fields.hubIncomingBackend,
-				lock:               tt.fields.lock,
-				Channels:           tt.fields.Channels,
-				Feeds:              tt.fields.Feeds,
-				Settings:           tt.fields.Settings,
-				NextUID:            tt.fields.NextUID,
-				Me:                 tt.fields.Me,
-				TokenEndpoint:      tt.fields.TokenEndpoint,
-				AuthEnabled:        tt.fields.AuthEnabled,
-				ticker:             tt.fields.ticker,
-				quit:               tt.fields.quit,
-				broker:             tt.fields.broker,
-				pool:               tt.fields.pool,
-			}
-			got, err := b.ChannelsCreate(tt.args.name)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("ChannelsCreate() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("ChannelsCreate() got = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
+// func Test_memoryBackend_ChannelsCreate(t *testing.T) {
+// 	type fields struct {
+// 		hubIncomingBackend hubIncomingBackend
+// 		lock               sync.RWMutex
+// 		Channels           map[string]microsub.Channel
+// 		Feeds              map[string][]microsub.Feed
+// 		Settings           map[string]channelSetting
+// 		NextUID            int
+// 		Me                 string
+// 		TokenEndpoint      string
+// 		AuthEnabled        bool
+// 		ticker             *time.Ticker
+// 		quit               chan struct{}
+// 		broker             *sse.Broker
+// 		pool               *redis.Pool
+// 	}
+// 	type args struct {
+// 		name string
+// 	}
+// 	tests := []struct {
+// 		name    string
+// 		fields  fields
+// 		args    args
+// 		want    microsub.Channel
+// 		wantErr bool
+// 	}{
+// 		{
+// 			name: "Duplicate channel",
+// 			fields: fields{
+// 				hubIncomingBackend: hubIncomingBackend{},
+// 				lock:               sync.RWMutex{},
+// 				Channels: func() map[string]microsub.Channel {
+// 					channels := make(map[string]microsub.Channel)
+// 					channels["1234"] = microsub.Channel{
+// 						UID:  "1234",
+// 						Name: "Test",
+// 						Unread: microsub.Unread{
+// 							Type:        microsub.UnreadCount,
+// 							Unread:      false,
+// 							UnreadCount: 0,
+// 						},
+// 					}
+// 					return channels
+// 				}(),
+// 				Feeds: func() map[string][]microsub.Feed {
+// 					feeds := make(map[string][]microsub.Feed)
+// 					return feeds
+// 				}(),
+// 				Settings:      nil,
+// 				NextUID:       1,
+// 				Me:            "",
+// 				TokenEndpoint: "",
+// 				AuthEnabled:   false,
+// 				ticker:        nil,
+// 				quit:          nil,
+// 				broker:        nil,
+// 				pool:          nil,
+// 			},
+// 			args: args{
+// 				name: "Test",
+// 			},
+// 			want: microsub.Channel{
+// 				UID:  "1234",
+// 				Name: "Test",
+// 				Unread: microsub.Unread{
+// 					Type:        microsub.UnreadCount,
+// 					Unread:      false,
+// 					UnreadCount: 0,
+// 				},
+// 			},
+// 			wantErr: false,
+// 		},
+// 	}
+// 	for _, tt := range tests {
+// 		t.Run(tt.name, func(t *testing.T) {
+// 			b := &memoryBackend{
+// 				hubIncomingBackend: tt.fields.hubIncomingBackend,
+// 				lock:               tt.fields.lock,
+// 				Channels:           tt.fields.Channels,
+// 				Feeds:              tt.fields.Feeds,
+// 				Settings:           tt.fields.Settings,
+// 				NextUID:            tt.fields.NextUID,
+// 				Me:                 tt.fields.Me,
+// 				TokenEndpoint:      tt.fields.TokenEndpoint,
+// 				AuthEnabled:        tt.fields.AuthEnabled,
+// 				ticker:             tt.fields.ticker,
+// 				quit:               tt.fields.quit,
+// 				broker:             tt.fields.broker,
+// 				pool:               tt.fields.pool,
+// 			}
+// 			got, err := b.ChannelsCreate(tt.args.name)
+// 			if (err != nil) != tt.wantErr {
+// 				t.Errorf("ChannelsCreate() error = %v, wantErr %v", err, tt.wantErr)
+// 				return
+// 			}
+// 			if !reflect.DeepEqual(got, tt.want) {
+// 				t.Errorf("ChannelsCreate() got = %v, want %v", got, tt.want)
+// 			}
+// 		})
+// 	}
+// }
+//
+// func Test_memoryBackend_removeFeed(t *testing.T) {
+// 	type fields struct {
+// 		Channels map[string]microsub.Channel
+// 		Feeds    map[string][]microsub.Feed
+// 	}
+// 	type args struct {
+// 		feedID string
+// 	}
+// 	tests := []struct {
+// 		name    string
+// 		fields  fields
+// 		args    args
+// 		lens    map[string]int
+// 		wantErr bool
+// 	}{
+// 		{
+// 			name: "remove from channel 1",
+// 			fields: fields{
+// 				Channels: map[string]microsub.Channel{
+// 					"123": {UID: "channel1", Name: "Channel 1"},
+// 					"124": {UID: "channel2", Name: "Channel 2"},
+// 				},
+// 				Feeds: map[string][]microsub.Feed{
+// 					"123": {{Type: "feed", URL: "feed1", Name: "Feed1"}},
+// 					"124": {{Type: "feed", URL: "feed2", Name: "Feed2"}},
+// 				},
+// 			},
+// 			args:    args{feedID: "feed1"},
+// 			lens:    map[string]int{"123": 0, "124": 1},
+// 			wantErr: false,
+// 		},
+// 		{
+// 			name: "remove from channel 2",
+// 			fields: fields{
+// 				Channels: map[string]microsub.Channel{
+// 					"123": {UID: "channel1", Name: "Channel 1"},
+// 					"124": {UID: "channel2", Name: "Channel 2"},
+// 				},
+// 				Feeds: map[string][]microsub.Feed{
+// 					"123": {{Type: "feed", URL: "feed1", Name: "Feed1"}},
+// 					"124": {{Type: "feed", URL: "feed2", Name: "Feed2"}},
+// 				},
+// 			},
+// 			args:    args{feedID: "feed2"},
+// 			lens:    map[string]int{"123": 1, "124": 0},
+// 			wantErr: false,
+// 		},
+// 		{
+// 			name: "remove unknown",
+// 			fields: fields{
+// 				Channels: map[string]microsub.Channel{
+// 					"123": {UID: "channel1", Name: "Channel 1"},
+// 					"124": {UID: "channel2", Name: "Channel 2"},
+// 				},
+// 				Feeds: map[string][]microsub.Feed{
+// 					"123": {{Type: "feed", URL: "feed1", Name: "Feed1"}},
+// 					"124": {{Type: "feed", URL: "feed2", Name: "Feed2"}},
+// 				},
+// 			},
+// 			args:    args{feedID: "feed3"},
+// 			lens:    map[string]int{"123": 1, "124": 1},
+// 			wantErr: false,
+// 		},
+// 		{
+// 			name: "remove from 0 channels",
+// 			fields: fields{
+// 				Channels: map[string]microsub.Channel{},
+// 				Feeds:    map[string][]microsub.Feed{},
+// 			},
+// 			args:    args{feedID: "feed3"},
+// 			lens:    map[string]int{},
+// 			wantErr: false,
+// 		},
+// 		{
+// 			name: "remove from multiple channels",
+// 			fields: fields{
+// 				Channels: map[string]microsub.Channel{
+// 					"123": {UID: "channel1", Name: "Channel 1"},
+// 					"124": {UID: "channel2", Name: "Channel 2"},
+// 				},
+// 				Feeds: map[string][]microsub.Feed{
+// 					"123": {{Type: "feed", URL: "feed1", Name: "Feed1"}},
+// 					"124": {{Type: "feed", URL: "feed1", Name: "Feed1"}},
+// 				},
+// 			},
+// 			args:    args{feedID: "feed1"},
+// 			lens:    map[string]int{"123": 0, "124": 0},
+// 			wantErr: false,
+// 		},
+// 	}
+// 	for _, tt := range tests {
+// 		t.Run(tt.name, func(t *testing.T) {
+// 			b := &memoryBackend{
+// 				Channels: tt.fields.Channels,
+// 				Feeds:    tt.fields.Feeds,
+// 			}
+// 			if err := b.removeFeed(tt.args.feedID); (err != nil) != tt.wantErr {
+// 				t.Errorf("removeFeed() error = %v, wantErr %v", err, tt.wantErr)
+// 			}
+// 			assert.Len(t, b.Channels, len(tt.lens))
+// 			for k, v := range tt.lens {
+// 				assert.Len(t, b.Feeds[k], v)
+// 			}
+// 		})
+// 	}
+// }

cmd/eksterd/micropub.go

@@ -1,7 +1,26 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package main
 
 import (
 	"crypto/sha1"
+	"database/sql"
 	"encoding/json"
 	"fmt"
 	"log"
@@ -40,44 +59,57 @@ func (h *micropubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	err := r.ParseForm()
 	if err != nil {
-		http.Error(w, "bad request", 400)
+		http.Error(w, "bad request", http.StatusBadRequest)
 		return
 	}
 
 	if r.Method == http.MethodPost {
 		var channel string
 
-		channel, err = getChannelFromAuthorization(r, conn)
+		sourceID, channel, err := getChannelFromAuthorization(r, conn, h.Backend.database)
 		if err != nil {
 			log.Println(err)
-			http.Error(w, "unauthorized", 401)
+			http.Error(w, "unauthorized", http.StatusUnauthorized)
 			return
 		}
 
 		// no channel is found
 		if channel == "" {
-			http.Error(w, "bad request, unknown channel", 400)
+			http.Error(w, "bad request, unknown channel", http.StatusBadRequest)
 			return
 		}
 
+		// TODO: We could try to fill the Source of the Item with something, but what?
 		item, err := parseIncomingItem(r)
 		if err != nil {
-			http.Error(w, err.Error(), 400)
+			log.Println(err)
+			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
+		log.Printf("Item published: %s", item.Published)
+		if item.Published == "" {
+			item.Published = time.Now().Format(time.RFC3339)
+		}
 
 		item.Read = false
 		newID, err := generateItemID(conn, channel)
 		if err != nil {
-			http.Error(w, err.Error(), 500)
+			log.Println(err)
+			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
 		}
 		item.ID = newID
 
-		err = h.Backend.channelAddItemWithMatcher(channel, *item)
+		item.Source = &microsub.Source{
+			ID:   fmt.Sprintf("micropub:%d", sourceID),
+			Name: fmt.Sprintf("Source %d", sourceID),
+		}
+
+		_, err = h.Backend.channelAddItemWithMatcher(channel, *item)
 		if err != nil {
 			log.Printf("could not add item to channel %s: %v", channel, err)
 		}
+
 		err = h.Backend.updateChannelUnreadCount(channel)
 		if err != nil {
 			log.Printf("could not update channel unread content %s: %v", channel, err)
@@ -86,12 +118,14 @@ func (h *micropubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "application/json")
 
 		if err = json.NewEncoder(w).Encode(map[string]string{"ok": "1"}); err != nil {
-			http.Error(w, "internal server error", 500)
+			log.Println(err)
+			http.Error(w, "internal server error", http.StatusInternalServerError)
 		}
+
 		return
 	}
 
-	http.Error(w, "Method not allowed", 405)
+	http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
 }
 
 func generateItemID(conn redis.Conn, channel string) (string, error) {
@@ -103,52 +137,57 @@ func generateItemID(conn redis.Conn, channel string) (string, error) {
 }
 
 func parseIncomingItem(r *http.Request) (*microsub.Item, error) {
-	var item microsub.Item
-
 	contentType := r.Header.Get("content-type")
 
 	if contentType == "application/jf2+json" {
-		dec := json.NewDecoder(r.Body)
-		err := dec.Decode(&item)
-		if err != nil {
-			return nil, errors.Wrapf(err, "could not decode request body as jf2: %v", err)
+		var item microsub.Item
+		if err := json.NewDecoder(r.Body).Decode(&item); err != nil {
+			return nil, fmt.Errorf("could not decode request body as %q: %v", contentType, err)
 		}
+		return &item, nil
 	} else if contentType == "application/json" {
 		var mfItem microformats.Microformat
-		dec := json.NewDecoder(r.Body)
-		err := dec.Decode(&mfItem)
-		if err != nil {
-			return nil, errors.Wrapf(err, "could not decode request body as json: %v", err)
+		if err := json.NewDecoder(r.Body).Decode(&mfItem); err != nil {
+			return nil, fmt.Errorf("could not decode request body as %q: %v", contentType, err)
 		}
 		author := microsub.Card{}
-		var ok bool
-		item, ok = jf2.SimplifyMicroformatItem(&mfItem, author)
+		item, ok := jf2.SimplifyMicroformatItem(&mfItem, author)
 		if !ok {
 			return nil, fmt.Errorf("could not simplify microformat item to jf2")
 		}
+		return &item, nil
 	} else if contentType == "application/x-www-form-urlencoded" {
+		// TODO: improve handling of form-urlencoded
+		var item microsub.Item
 		content := r.FormValue("content")
 		name := r.FormValue("name")
 		item.Type = "entry"
 		item.Name = name
 		item.Content = &microsub.Content{Text: content}
 		item.Published = time.Now().Format(time.RFC3339)
-	} else {
-		return nil, fmt.Errorf("content-type %s is not supported", contentType)
+		return &item, nil
 	}
-	return &item, nil
+
+	return nil, fmt.Errorf("content-type %q is not supported", contentType)
 }
 
-func getChannelFromAuthorization(r *http.Request, conn redis.Conn) (string, error) {
+func getChannelFromAuthorization(r *http.Request, conn redis.Conn, database *sql.DB) (int, string, error) {
 	// backward compatible
 	sourceID := r.URL.Query().Get("source_id")
 	if sourceID != "" {
-		channel, err := redis.String(conn.Do("HGET", "sources", sourceID))
-		if err != nil {
-			return "", errors.Wrapf(err, "could not get channel for sourceID: %s", sourceID)
-		}
+		row := database.QueryRow(`
+SELECT s.id as source_id, c.uid
+FROM "sources" AS "s"
+INNER JOIN "channels" AS "c" ON s.channel_id = c.id
+WHERE "auth_code" = $1
+`, sourceID)
 
-		return channel, nil
+		var channel string
+		var sourceID int
+		if err := row.Scan(&sourceID, &channel); err == sql.ErrNoRows {
+			return 0, "", errors.New("channel not found")
+		}
+		return sourceID, channel, nil
 	}
 
 	// full micropub with indieauth
@@ -157,11 +196,11 @@ func getChannelFromAuthorization(r *http.Request, conn redis.Conn) (string, erro
 		token := authHeader[7:]
 		channel, err := redis.String(conn.Do("HGET", "token:"+token, "channel"))
 		if err != nil {
-			return "", errors.Wrap(err, "could not get channel for token")
+			return 0, "", errors.Wrap(err, "could not get channel for token")
 		}
 
-		return channel, nil
+		return 0, channel, nil
 	}
 
-	return "", fmt.Errorf("could not get channel from authorization")
+	return 0, "", fmt.Errorf("could not get channel from authorization")
 }

cmd/eksterd/search.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package main
 
 import (

cmd/eksterd/templates/auth.html

@@ -1,6 +1,5 @@
-{{- /*gotype: p83.nl/go/ekster/cmd/eksterd.authPage*/ -}}
 <!DOCTYPE html>
-<html lang="en">
+<html>
 <head>
 <meta charset="utf-8">
 <meta name="viewport" content="width=device-width, initial-scale=1">

cmd/eksterd/templates/channel.html

@@ -1,4 +1,3 @@
-{{- /*gotype: p83.nl/go/ekster/cmd/eksterd.settingsPage*/ -}}
 <!DOCTYPE html>
 <html>
 <head>

cmd/eksterd/templates/index.html

@@ -12,8 +12,6 @@
 <body>
     <section class="section">
         <div class="container">
-
-
             <nav class="navbar" role="navigation" aria-label="main navigation">
                 <div class="navbar-brand">
                     <a class="navbar-item" href="/">
@@ -65,6 +63,11 @@
                     </div>
                 </form>
             {{ end }}
+
+            <p>Copyright (C) 2018 The Ekster authors.<br>
+            This program comes with ABSOLUTELY NO WARRANTY. This is free software,
+            and you are welcome to redistribute it under certain conditions.
+            See <a href="https://www.gnu.org/licenses/gpl-3.0.en.html">Licence</a> for more details.</p>
         </div>
     </section>
 </body>

cmd/jf2test/main.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package main
 
 import (
@@ -32,7 +50,7 @@ func main() {
 	}
 	defer resp.Body.Close()
 
-	items, err := fetch.FeedItems(Fetch, url, resp.Header.Get("Content-Type"), resp.Body)
+	items, err := fetch.FeedItems(fetch.FetcherFunc(Fetch), url, resp.Header.Get("Content-Type"), resp.Body)
 	if err != nil {
 		log.Fatal(err)
 	}

docker-compose.yml

@@ -4,7 +4,7 @@ services:
     image: "redis:5"
 
   database:
-    image: postgres
+    image: postgres:14
     volumes:
     - database-data:/var/lib/postgresql/data
     environment:
@@ -14,21 +14,16 @@ services:
       POSTGRES_HOST_AUTH_METHOD: trust
 
   web:
-    image: "pstuifzand/ekster:alpine"
-    working_dir: /opt/microsub
-    links:
-      - redis:redis
+    image: ubuntu
+    working_dir: /app
     volumes:
-      - microsub-data:/opt/microsub
-      - ./templates:/app/templates
       - ./eksterd:/app/eksterd
+      - ./backend.json:/app/backend.json
     entrypoint: /app/eksterd
-    command: -auth=false -port 80 -templates templates
+    command: -auth=false -port 80
     ports:
       - 8089:80
     environment:
-      - "FEEDBIN_USER="
-      - "FEEDBIN_PASS="
       - "EKSTER_BASEURL=http://localhost:8089/"
       - "EKSTER_TEMPLATES=/app/templates"
 

go.mod

@@ -3,15 +3,16 @@ module p83.nl/go/ekster
 go 1.16
 
 require (
-	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751
 	github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394
-	github.com/blevesearch/bleve/v2 v2.0.3 // indirect
+	github.com/blevesearch/bleve/v2 v2.0.3
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/gilliek/go-opml v1.0.0
+	github.com/golang-migrate/migrate/v4 v4.15.1
 	github.com/gomodule/redigo v1.8.2
 	github.com/lib/pq v1.10.1
 	github.com/pkg/errors v0.9.1
-	github.com/stretchr/testify v1.5.1
-	golang.org/x/net v0.0.0-20200707034311-ab3426394381
+	github.com/stretchr/testify v1.7.0
+	golang.org/x/net v0.0.0-20211013171255-e13a2654a71e
+	golang.org/x/text v0.3.7
 	willnorris.com/go/microformats v1.1.0
 )

pkg/auth/types.go

@@ -1,6 +1,24 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package auth
 
-// Auther
+// Auther checks of the token in the head is accepted and fills TokenResponse
 type Auther interface {
 	AuthTokenAccepted(header string, r *TokenResponse) bool
 }

pkg/client/requests.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package client
 
 import (
@@ -88,7 +106,7 @@ func (c *Client) microsubPostRequest(action string, args map[string]string) (*ht
 
 	if res.StatusCode != 200 {
 		msg, _ := ioutil.ReadAll(res.Body)
-		return nil, fmt.Errorf("unsuccessful response: %d: %q", res.StatusCode, string(msg))
+		return nil, fmt.Errorf("unsuccessful response: %d: %q", res.StatusCode, strings.TrimSpace(string(msg)))
 	}
 
 	return res, err
@@ -116,7 +134,7 @@ func (c *Client) microsubPostFormRequest(action string, args map[string]string,
 
 	if res.StatusCode != 200 {
 		msg, _ := ioutil.ReadAll(res.Body)
-		return nil, fmt.Errorf("unsuccessful response: %d: %q", res.StatusCode, string(msg))
+		return nil, fmt.Errorf("unsuccessful response: %d: %q", res.StatusCode, strings.TrimSpace(string(msg)))
 	}
 
 	return res, err
@@ -180,7 +198,7 @@ func (c *Client) TimelineGet(before, after, channel string) (microsub.Timeline,
 func (c *Client) PreviewURL(url string) (microsub.Timeline, error) {
 	args := make(map[string]string)
 	args["url"] = url
-	res, err := c.microsubGetRequest("preview", args)
+	res, err := c.microsubPostRequest("preview", args)
 	if err != nil {
 		return microsub.Timeline{}, err
 	}

pkg/fetch/fetch.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 // Package fetch provides an API for fetching information about urls.
 package fetch
 
@@ -25,7 +43,7 @@ import (
 )
 
 // FeedHeader returns a new microsub.Feed with the information parsed from body.
-func FeedHeader(fetcher FetcherFunc, fetchURL, contentType string, body io.Reader) (microsub.Feed, error) {
+func FeedHeader(fetcher Fetcher, fetchURL, contentType string, body io.Reader) (microsub.Feed, error) {
 	log.Printf("ProcessContent %s\n", fetchURL)
 	log.Println("Found " + contentType)
 
@@ -38,7 +56,7 @@ func FeedHeader(fetcher FetcherFunc, fetchURL, contentType string, body io.Reade
 		author, ok := jf2.SimplifyMicroformatDataAuthor(data)
 		if !ok {
 			if strings.HasPrefix(author.URL, "http") {
-				resp, err := fetcher(author.URL)
+				resp, err := fetcher.Fetch(author.URL)
 				if err != nil {
 					return feed, err
 				}
@@ -48,6 +66,9 @@ func FeedHeader(fetcher FetcherFunc, fetchURL, contentType string, body io.Reade
 				md := microformats.Parse(resp.Body, u)
 
 				author, ok = jf2.SimplifyMicroformatDataAuthor(md)
+				if !ok {
+					log.Println("Could not simplify the author")
+				}
 			}
 		}
 
@@ -108,7 +129,7 @@ func FeedHeader(fetcher FetcherFunc, fetchURL, contentType string, body io.Reade
 }
 
 // FeedItems returns the items from the url, parsed from body.
-func FeedItems(fetcher FetcherFunc, fetchURL, contentType string, body io.Reader) ([]microsub.Item, error) {
+func FeedItems(fetcher Fetcher, fetchURL, contentType string, body io.Reader) ([]microsub.Item, error) {
 	log.Printf("ProcessContent %s\n", fetchURL)
 	log.Println("Found " + contentType)
 

pkg/fetch/fetch_test.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package fetch
 
 import (
@@ -23,7 +41,7 @@ func TestFeedHeader(t *testing.T) {
 </body>
 </html>
 `
-	feed, err := FeedHeader(fetcher, "https://example.com/", "text/html", strings.NewReader(doc))
+	feed, err := FeedHeader(FetcherFunc(fetcher), "https://example.com/", "text/html", strings.NewReader(doc))
 	if assert.NoError(t, err) {
 		assert.Equal(t, "feed", feed.Type)
 		assert.Equal(t, "Title", feed.Name)

pkg/fetch/fetcher.go

@@ -1,6 +1,34 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package fetch
 
 import "net/http"
 
+// Fetcher fetches urls
+type Fetcher interface {
+	Fetch(url string) (*http.Response, error)
+}
+
 // FetcherFunc is a function that fetches an url
 type FetcherFunc func(url string) (*http.Response, error)
+
+// Fetch fetches an url and returns a response or error
+func (ff FetcherFunc) Fetch(url string) (*http.Response, error) {
+	return ff(url)
+}

pkg/indieauth/auth.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package indieauth
 
 import (

pkg/jf2/reflect.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package jf2
 
 import (

pkg/jf2/simplify.go

@@ -1,21 +1,22 @@
-// Package jf2 converts microformats to JF2
 /*
-   ekster - microsub server
-   Copyright (C) 2018  Peter Stuifzand
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
 
-   This program is free software: you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published by
-   the Free Software Foundation, either version 3 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
+// Package jf2 converts microformats to JF2
 package jf2
 
 import (
@@ -88,6 +89,14 @@ func simplifyContent(k string, v []interface{}) *microsub.Content {
 // CleanHTML removes white-space:pre from html
 func CleanHTML(s string) (string, error) {
 	doc, err := html.Parse(strings.NewReader(s))
+	if err != nil {
+		return "", err
+	}
+
+	whitespaceRegex, err := regexp.Compile(`white-space:\s*pre`)
+	if err != nil {
+		return "", err
+	}
 
 	if err != nil {
 		return "", err
@@ -101,7 +110,7 @@ func CleanHTML(s string) (string, error) {
 				if a.Key != "style" {
 					continue
 				}
-				if m, err := regexp.MatchString("white-space:\\s*pre", a.Val); err == nil && m {
+				if whitespaceRegex.MatchString(a.Val) {
 					removeIndex = i
 					break
 				}

pkg/jf2/simplify_test.go

@@ -1,20 +1,20 @@
 /*
-   ekster - microsub server
-   Copyright (C) 2018  Peter Stuifzand
-
-   This program is free software: you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published by
-   the Free Software Foundation, either version 3 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
 package jf2_test
 
 import (

pkg/jsonfeed/jsonfeed.go

@@ -1,18 +1,22 @@
-// Package jsonfeed contains the types and a parse function for JSON feeds.
-// Copyright (C) 2018  Peter Stuifzand
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// Package jsonfeed parses feeds in the jsonfeed format
 package jsonfeed
 
 import (

pkg/jsonfeed/jsonfeed_test.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package jsonfeed
 
 import (

pkg/microsub/database.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package microsub
 
 import (

pkg/microsub/protocol.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 // Package microsub describes the protocol methods of the Microsub protocol
 package microsub
 
@@ -82,6 +100,15 @@ type Item struct {
 	Refs       map[string]Item `json:"refs,omitempty"`
 	ID         string          `json:"_id,omitempty"`
 	Read       bool            `json:"_is_read"`
+	Source     *Source         `json:"_source,omitempty"`
+}
+
+// Source is an Item source
+type Source struct {
+	ID    string `json:"_id"`
+	URL   string `json:"url"`
+	Name  string `json:"name"`
+	Photo string `json:"photo"`
 }
 
 // Pagination contains information about paging

pkg/microsub/protocol_test.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package microsub
 
 import (

pkg/rss/ISO-8859-1.go

@@ -37,7 +37,7 @@ func (cs *charsetISO88591er) ReadByte() (b byte, err error) {
 
 func (cs *charsetISO88591er) Read(p []byte) (int, error) {
 	// Use ReadByte method.
-	return 0, errors.New("Use ReadByte()")
+	return 0, errors.New("use ReadByte()")
 }
 
 func isCharsetISO88591(charset string) bool {

pkg/rss/rss.go

@@ -303,7 +303,7 @@ type Enclosure struct {
 // Get uses http.Get to fetch an enclosure.
 func (e *Enclosure) Get() (io.ReadCloser, error) {
 	if e == nil || e.URL == "" {
-		return nil, errors.New("No enclosure")
+		return nil, errors.New("no enclosure")
 	}
 
 	res, err := http.Get(e.URL)
@@ -325,7 +325,7 @@ type Image struct {
 // Get uses http.Get to fetch an image.
 func (i *Image) Get() (io.ReadCloser, error) {
 	if i == nil || i.URL == "" {
-		return nil, errors.New("No image")
+		return nil, errors.New("no image")
 	}
 
 	res, err := http.Get(i.URL)

pkg/rss/rss_1.0.go

@@ -5,8 +5,10 @@ import (
 	"encoding/xml"
 	"fmt"
 	"sort"
-	"strings"
 	"time"
+
+	"golang.org/x/text/cases"
+	"golang.org/x/text/language"
 )
 
 func parseRSS1(data []byte) (*Feed, error) {
@@ -29,6 +31,8 @@ func parseRSS1(data []byte) (*Feed, error) {
 	out.Description = channel.Description
 	out.Link = channel.Link
 	out.Image = channel.Image.Image()
+
+	titleCaser := cases.Title(language.English)
 	if channel.MinsToLive != 0 {
 		sort.Ints(channel.SkipHours)
 		next := time.Now().Add(time.Duration(channel.MinsToLive) * time.Minute)
@@ -41,7 +45,7 @@ func parseRSS1(data []byte) (*Feed, error) {
 		for trying {
 			trying = false
 			for _, day := range channel.SkipDays {
-				if strings.Title(day) == next.Weekday().String() {
+				if titleCaser.String(day) == next.Weekday().String() {
 					next.Add(time.Duration(24-next.Hour()) * time.Hour)
 					trying = true
 					break

pkg/rss/rss_2.0.go

@@ -5,8 +5,10 @@ import (
 	"encoding/xml"
 	"fmt"
 	"sort"
-	"strings"
 	"time"
+
+	"golang.org/x/text/cases"
+	"golang.org/x/text/language"
 )
 
 func parseRSS2(data []byte) (*Feed, error) {
@@ -38,6 +40,7 @@ func parseRSS2(data []byte) (*Feed, error) {
 
 	out.Image = channel.Image.Image()
 	if channel.MinsToLive != 0 {
+		titleCaser := cases.Title(language.English)
 		sort.Ints(channel.SkipHours)
 		next := time.Now().Add(time.Duration(channel.MinsToLive) * time.Minute)
 		for _, hour := range channel.SkipHours {
@@ -49,7 +52,7 @@ func parseRSS2(data []byte) (*Feed, error) {
 		for trying {
 			trying = false
 			for _, day := range channel.SkipDays {
-				if strings.Title(day) == next.Weekday().String() {
+				if titleCaser.String(day) == next.Weekday().String() {
 					next.Add(time.Duration(24-next.Hour()) * time.Hour)
 					trying = true
 					break

pkg/rss/rss_2.0_test.go

@@ -1,7 +1,6 @@
 package rss
 
 import (
-	"fmt"
 	"io/ioutil"
 	"path/filepath"
 	"testing"
@@ -77,7 +76,7 @@ func TestParseItemDateOK(t *testing.T) {
 			t.Fatalf("Parsing %s: %v", name, err)
 		}
 
-		if fmt.Sprintf("%s", feed.Items[0].Date) != want {
+		if feed.Items[0].Date.String() != want {
 			t.Errorf("%s: got %q, want %q", name, feed.Items[0].Date, want)
 		}
 	}
@@ -100,7 +99,7 @@ func TestParseItemDateFailure(t *testing.T) {
 			t.Fatalf("Parsing %s: %v", name, err)
 		}
 
-		if fmt.Sprintf("%s", feed.Items[1].Date) != want {
+		if feed.Items[1].Date.String() != want {
 			t.Errorf("%s: got %q, want %q", name, feed.Items[1].Date, want)
 		}
 

pkg/server/microsub.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 /*
 Package server contains the microsub server itself. It implements http.Handler.
 It follows the spec at https://indieweb.org/Microsub-spec.
@@ -16,7 +34,7 @@ import (
 )
 
 var (
-	entryRegex = regexp.MustCompile("^entry\\[\\d+\\]$")
+	entryRegex = regexp.MustCompile(`^entry\[\d+\]$`)
 )
 
 // Constants used for the responses
@@ -36,7 +54,7 @@ func respondJSON(w http.ResponseWriter, value interface{}) {
 	w.Header().Add("Content-Type", OutputContentType)
 	err := jw.Encode(value)
 	if err != nil {
-		http.Error(w, err.Error(), 500)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
 	}
 }
 
@@ -69,7 +87,8 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		if action == "channels" {
 			channels, err := h.backend.ChannelsGetList()
 			if err != nil {
-				http.Error(w, err.Error(), 500)
+				log.Println(err)
+				http.Error(w, err.Error(), http.StatusInternalServerError)
 				return
 			}
 			respondJSON(w, map[string][]microsub.Channel{
@@ -78,14 +97,16 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		} else if action == "timeline" {
 			timeline, err := h.backend.TimelineGet(values.Get("before"), values.Get("after"), values.Get("channel"))
 			if err != nil {
-				http.Error(w, err.Error(), 500)
+				log.Println(err)
+				http.Error(w, err.Error(), http.StatusInternalServerError)
 				return
 			}
 			respondJSON(w, timeline)
 		} else if action == "preview" {
 			timeline, err := h.backend.PreviewURL(values.Get("url"))
 			if err != nil {
-				http.Error(w, err.Error(), 500)
+				log.Println(err)
+				http.Error(w, err.Error(), http.StatusInternalServerError)
 				return
 			}
 			respondJSON(w, timeline)
@@ -93,7 +114,8 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			channel := values.Get("channel")
 			following, err := h.backend.FollowGetList(channel)
 			if err != nil {
-				http.Error(w, err.Error(), 500)
+				log.Println(err)
+				http.Error(w, err.Error(), http.StatusInternalServerError)
 				return
 			}
 			respondJSON(w, map[string][]microsub.Feed{
@@ -102,7 +124,8 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		} else if action == "events" {
 			events, err := h.backend.Events()
 			if err != nil {
-				http.Error(w, "could not start sse connection", 500)
+				log.Println(err)
+				http.Error(w, "could not start sse connection", http.StatusInternalServerError)
 			}
 
 			// Remove this client from the map of connected clients
@@ -112,7 +135,7 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			}()
 
 			// Listen to connection close and un-register messageChan
-			notify := w.(http.CloseNotifier).CloseNotify()
+			notify := r.Context().Done()
 
 			go func() {
 				<-notify
@@ -122,10 +145,10 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			err = sse.WriteMessages(w, events)
 			if err != nil {
 				log.Println(err)
-				http.Error(w, "internal server error", 500)
+				http.Error(w, "internal server error", http.StatusInternalServerError)
 			}
 		} else {
-			http.Error(w, fmt.Sprintf("unknown action %s\n", action), 400)
+			http.Error(w, fmt.Sprintf("unknown action %s", action), http.StatusBadRequest)
 			return
 		}
 		return
@@ -141,7 +164,8 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			if method == "delete" {
 				err := h.backend.ChannelsDelete(uid)
 				if err != nil {
-					http.Error(w, err.Error(), 500)
+					log.Println(err)
+					http.Error(w, err.Error(), http.StatusInternalServerError)
 					return
 				}
 				respondJSON(w, []string{})
@@ -151,14 +175,16 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			if uid == "" {
 				channel, err := h.backend.ChannelsCreate(name)
 				if err != nil {
-					http.Error(w, err.Error(), 500)
+					log.Println(err)
+					http.Error(w, err.Error(), http.StatusInternalServerError)
 					return
 				}
 				respondJSON(w, channel)
 			} else if name != "" {
 				channel, err := h.backend.ChannelsUpdate(uid, name)
 				if err != nil {
-					http.Error(w, err.Error(), 500)
+					log.Println(err)
+					http.Error(w, err.Error(), http.StatusInternalServerError)
 					return
 				}
 				respondJSON(w, channel)
@@ -169,7 +195,7 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			// h.HubIncomingBackend.CreateFeed(url, uid)
 			feed, err := h.backend.FollowURL(uid, url)
 			if err != nil {
-				http.Error(w, err.Error(), 500)
+				http.Error(w, err.Error(), http.StatusInternalServerError)
 				return
 			}
 			respondJSON(w, feed)
@@ -178,10 +204,17 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			url := values.Get("url")
 			err := h.backend.UnfollowURL(uid, url)
 			if err != nil {
-				http.Error(w, err.Error(), 500)
+				http.Error(w, err.Error(), http.StatusInternalServerError)
 				return
 			}
 			respondJSON(w, []string{})
+		} else if action == "preview" {
+			timeline, err := h.backend.PreviewURL(values.Get("url"))
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+				return
+			}
+			respondJSON(w, timeline)
 		} else if action == "search" {
 			query := values.Get("query")
 			channel := values.Get("channel")
@@ -206,6 +239,7 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 					})
 					return
 				}
+				log.Printf("Searching for %s in %s (%d results)", query, channel, len(items))
 				respondJSON(w, map[string]interface{}{
 					"query": query,
 					"items": items,
@@ -235,22 +269,21 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 				if len(markAsRead) > 0 {
 					err := h.backend.MarkRead(channel, markAsRead)
 					if err != nil {
-						http.Error(w, err.Error(), 500)
+						http.Error(w, err.Error(), http.StatusInternalServerError)
 						return
 					}
 				} else {
 					log.Println("No uids specified for mark read")
 				}
 			} else {
-				http.Error(w, fmt.Sprintf("unknown method in timeline %s\n", method), 500)
+				http.Error(w, fmt.Sprintf("unknown method in timeline %s\n", method), http.StatusInternalServerError)
 				return
 			}
 
 			respondJSON(w, []string{})
 		} else {
-			http.Error(w, fmt.Sprintf("unknown action %s\n", action), 400)
+			http.Error(w, fmt.Sprintf("unknown action %s\n", action), http.StatusBadRequest)
 		}
 		return
 	}
-	return
 }

pkg/server/microsub_test.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package server
 
 import (

pkg/server/null.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package server
 
 import (

pkg/sse/events.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package sse
 
 import (

pkg/timeline/null.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package timeline
 
 import "p83.nl/go/ekster/pkg/microsub"

pkg/timeline/postgres.go

@@ -1,10 +1,31 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package timeline
 
 import (
 	"context"
+	"crypto/sha256"
 	"database/sql"
+	"encoding/hex"
 	"fmt"
 	"log"
+	"strconv"
 	"strings"
 	"time"
 
@@ -33,46 +54,10 @@ func (p *postgresStream) Init() error {
 		return fmt.Errorf("database ping failed: %w", err)
 	}
 
-	_, err = conn.ExecContext(ctx, `
-CREATE TABLE IF NOT EXISTS "channels" (
-    "id" int primary key generated always as identity,
-    "name" varchar(255) unique,
-    "created_at" timestamp DEFAULT current_timestamp
-);
-`)
-	if err != nil {
-		return fmt.Errorf("create channels table failed: %w", err)
-	}
-
-	_, err = conn.ExecContext(ctx, `
-CREATE TABLE IF NOT EXISTS "items" (
-    "id" int primary key generated always as identity,
-    "channel_id" int references "channels" on delete cascade,
-    "uid" varchar(512) not null unique,
-    "is_read" int default 0,
-    "data" json,
-    "created_at" timestamp DEFAULT current_timestamp,
-    "updated_at" timestamp,
-    "published_at" timestamp
-);
-`)
-	if err != nil {
-		return fmt.Errorf("create items table failed: %w", err)
-	}
-
-	_, err = conn.ExecContext(ctx, `INSERT INTO "channels" ("name", "created_at") VALUES ($1, DEFAULT)
- 		ON CONFLICT DO NOTHING`, p.channel)
-	if err != nil {
-		return fmt.Errorf("create channel failed: %w", err)
-	}
-
-	row := conn.QueryRowContext(ctx, `SELECT "id" FROM "channels" WHERE "name" = $1`, p.channel)
-	if row == nil {
-		return fmt.Errorf("fetch channel failed: %w", err)
-	}
+	row := conn.QueryRowContext(ctx, `SELECT "id" FROM "channels" WHERE "uid" = $1`, p.channel)
 	err = row.Scan(&p.channelID)
-	if err != nil {
-		return fmt.Errorf("fetch channel failed while scanning: %w", err)
+	if err == sql.ErrNoRows {
+		return fmt.Errorf("channel %s not found: %w", p.channel, err)
 	}
 
 	return nil
@@ -101,16 +86,16 @@ WHERE "channel_id" = $1
 			log.Println(err)
 		} else {
 			args = append(args, b)
-			qb.WriteString(` AND "published_at" < $2`)
+			qb.WriteString(` AND "published_at" > $2`)
 		}
 	} else if after != "" {
 		b, err := time.Parse(time.RFC3339, after)
 		if err == nil {
 			args = append(args, b)
-			qb.WriteString(` AND "published_at" > $2`)
+			qb.WriteString(` AND "published_at" < $2`)
 		}
 	}
-	qb.WriteString(` ORDER BY "published_at" DESC LIMIT 10`)
+	qb.WriteString(` ORDER BY "published_at" DESC LIMIT 20`)
 
 	rows, err := conn.QueryContext(context.Background(), qb.String(), args...)
 	if err != nil {
@@ -154,9 +139,12 @@ WHERE "channel_id" = $1
 		return tl, err
 	}
 
-	// TODO: should only be set of there are more items available
-	tl.Paging.Before = last
-	// tl.Paging.After = last
+	if len(tl.Items) > 0 && hasMoreBefore(conn, tl.Items[0].Published) {
+		tl.Paging.Before = tl.Items[0].Published
+	}
+	if hasMoreAfter(conn, last) {
+		tl.Paging.After = last
+	}
 
 	if tl.Items == nil {
 		tl.Items = []microsub.Item{}
@@ -165,6 +153,24 @@ WHERE "channel_id" = $1
 	return tl, nil
 }
 
+func hasMoreBefore(conn *sql.Conn, before string) bool {
+	row := conn.QueryRowContext(context.Background(), `SELECT COUNT(*) FROM "items" WHERE "published_at" > $1`, before)
+	var count int
+	if err := row.Scan(&count); err == sql.ErrNoRows {
+		return false
+	}
+	return count > 0
+}
+
+func hasMoreAfter(conn *sql.Conn, after string) bool {
+	row := conn.QueryRowContext(context.Background(), `SELECT COUNT(*) FROM "items" WHERE "published_at" < $1`, after)
+	var count int
+	if err := row.Scan(&count); err == sql.ErrNoRows {
+		return false
+	}
+	return count > 0
+}
+
 // Count
 func (p *postgresStream) Count() (int, error) {
 	ctx := context.Background()
@@ -173,16 +179,12 @@ func (p *postgresStream) Count() (int, error) {
 		return -1, err
 	}
 	defer conn.Close()
+	var count int
 	row := conn.QueryRowContext(context.Background(), `SELECT COUNT(*) FROM items WHERE channel_id = $1 AND "is_read" = 0`, p.channelID)
-	if row == nil {
+	err = row.Scan(&count)
+	if err != nil && err == sql.ErrNoRows {
 		return 0, nil
 	}
-	var count int
-	err = row.Scan(&count)
-	if err != nil {
-		return -1, err
-	}
-
 	return count, nil
 }
 
@@ -203,14 +205,34 @@ func (p *postgresStream) AddItem(item microsub.Item) (bool, error) {
 		}
 		t = t2
 	}
+	if item.ID == "" {
+		// FIXME: This won't work when we receive the item multiple times
+		h := sha256.Sum256([]byte(fmt.Sprintf("%s:%d", p.channel, time.Now().UnixNano())))
+		item.UID = hex.EncodeToString(h[:])
+	}
+
+	var optFeedID sql.NullInt64
+	if item.Source == nil || item.Source.ID == "" {
+		optFeedID.Valid = false
+		optFeedID.Int64 = 0
+	} else {
+		feedID, err := strconv.ParseInt(item.Source.ID, 10, 64)
+		if err != nil {
+			optFeedID.Valid = false
+			optFeedID.Int64 = 0
+		} else {
+			optFeedID.Valid = true
+			optFeedID.Int64 = feedID
+		}
+	}
 
 	result, err := conn.ExecContext(context.Background(), `
-INSERT INTO "items" ("channel_id", "uid", "data", "published_at", "created_at")
-VALUES ($1, $2, $3, $4, DEFAULT)
+INSERT INTO "items" ("channel_id", "feed_id", "uid", "data", "published_at", "created_at")
+VALUES ($1, $2, $3, $4, $5, DEFAULT)
 ON CONFLICT ON CONSTRAINT "items_uid_key" DO NOTHING
-`, p.channelID, item.ID, &item, t)
+`, p.channelID, optFeedID, item.ID, &item, t)
 	if err != nil {
-		return false, fmt.Errorf("while adding item: %w", err)
+		return false, fmt.Errorf("insert item: %w", err)
 	}
 	c, err := result.RowsAffected()
 	if err != nil {

pkg/timeline/redisset.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package timeline
 
 import (
@@ -114,6 +132,12 @@ func (timeline *redisSortedSetTimeline) AddItem(item microsub.Item) (bool, error
 		item.Published = time.Now().Format(time.RFC3339)
 	}
 
+	// Fix date when it almost matches with RFC3339, except the colon in the timezone
+	format := "2006-01-02T15:04:05Z0700"
+	if parsedDate, err := time.Parse(format, item.Published); err == nil {
+		item.Published = parsedDate.Format(time.RFC3339)
+	}
+
 	data, err := json.Marshal(item)
 	if err != nil {
 		return false, fmt.Errorf("couldn't marshal item for redis: %s", err)

pkg/timeline/redisstreams.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package timeline
 
 import (

pkg/timeline/timeline.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 // Package timeline contains different types of timeline backends.
 //
 // "sorted-set" uses Redis sorted sets as a backend

pkg/util/algorithm.go

@@ -1,7 +1,26 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package util
 
 import "reflect"
 
+// Rotate rotates values of an array, between index f and l with midpoint k
 func Rotate(a interface{}, f, k, l int) int {
 	swapper := reflect.Swapper(a)
 	if f == k {
@@ -38,6 +57,7 @@ func Rotate(a interface{}, f, k, l int) int {
 	return ret
 }
 
+// StablePartition partitions elements of the array between indices f and l according to predicate p
 func StablePartition(a interface{}, f, l int, p func(i int) bool) int {
 	n := l - f
 
@@ -48,7 +68,7 @@ func StablePartition(a interface{}, f, l int, p func(i int) bool) int {
 	if n == 1 {
 		t := f
 		if p(f) {
-			t += 1
+			t++
 		}
 		return t
 	}

pkg/util/randkey.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package util
 
 import (
@@ -6,6 +24,7 @@ import (
 
 const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
 
+// RandStringBytes generates a random string of n characters
 func RandStringBytes(n int) string {
 	b := make([]byte, n)
 	for i := range b {

pkg/websub/signature.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package websub
 
 import (

pkg/websub/signature_test.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package websub
 
 import (

pkg/websub/subscribe.go

@@ -1,3 +1,21 @@
+/*
+ *  Ekster is a microsub server
+ *  Copyright (c) 2021 The Ekster authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package websub
 
 import (

templates/auth.html

@@ -1,100 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-<meta charset="utf-8">
-<meta name="viewport" content="width=device-width, initial-scale=1">
-<title>Ekster</title>
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.1/css/bulma.min.css">
-</head>
-<body>
-    <section class="section">
-        <div class="container">
-
-
-            <nav class="navbar" role="navigation" aria-label="main navigation">
-                <div class="navbar-brand">
-                    <a class="navbar-item" href="/">
-                        Ekster
-                    </a>
-
-                    <a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false" data-target="menu">
-                        <span aria-hidden="true"></span>
-                        <span aria-hidden="true"></span>
-                        <span aria-hidden="true"></span>
-                    </a>
-                </div>
-
-                {{ if .Session.LoggedIn }}
-                    <div id="menu" class="navbar-menu">
-                        <a class="navbar-item" href="/settings">
-                            Settings
-                        </a>
-                        <a class="navbar-item" href="/logs">
-                            Logs
-                        </a>
-                        <a class="navbar-item" href="{{ .Session.Me }}">
-                            Profile
-                        </a>
-                    </div>
-                {{ end }}
-            </nav>
-
-            <h1 class="title">Ekster - Microsub server</h1>
-
-            <div class="box">
-                <form action="/auth/approve" method="post">
-                    <input type="hidden" name="state" value="{{ .State }}" />
-
-                    <div class="field">
-                        <label class="label">Client ID</label>
-                        <div class="control">
-                            <p>{{ .ClientID }}</p>
-                        </div>
-                        <div class="control">
-                            <p>{{ .App.Name }}</p>
-                        </div>
-                        <div class="control">
-                            <p><img src="{{ .App.IconURL }}" /></p>
-                        </div>
-                    </div>
-
-                    <div class="field">
-                        <label class="label">RedirectURI</label>
-                        <div class="control">
-                            <p>{{ .RedirectURI }}</p>
-                        </div>
-                    </div>
-                    
-                    <div class="field">
-                        <label class="label">Scope</label>
-                        <div class="control">
-                            <p>{{ .Scope }}</p>
-                        </div>
-                    </div>
-
-                    <div class="field">
-                        <label class="label">Select a channel</label>
-                        <div class="control">
-                            <div class="select">
-                                <select name="channel">
-                                    {{ range .Channels }}
-                                        <option value="{{ .UID }}">{{ .Name }}</option>
-                                    {{ end }}
-                                </select>
-                            </div>
-                        </div>
-                    </div>
-
-                    <div class="field">
-                        <div class="control">
-                            <button type="submit" name="accept" value="approve" class="button is-primary">
-                                Approve
-                            </button>
-                        </div>
-                    </div>
-                </form>
-            </div>
-        </div>
-    </section>
-</body>
-</html>

templates/base.html

@@ -1,49 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <title>{{template "title" .}}</title>
-    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.1/css/bulma.min.css">
-</head>
-<body>
-<section class="section">
-    <div class="container">
-        <nav class="navbar" role="navigation" aria-label="main navigation">
-            <div class="navbar-brand">
-                <a class="navbar-item" href="/">
-                    Ekster
-                </a>
-
-                <a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false" data-target="menu">
-                    <span aria-hidden="true"></span>
-                    <span aria-hidden="true"></span>
-                    <span aria-hidden="true"></span>
-                </a>
-            </div>
-
-            {{ if .Session.LoggedIn }}
-                <div id="menu" class="navbar-menu">
-                    <a class="navbar-item" href="/settings">
-                        Settings
-                    </a>
-                    <a class="navbar-item" href="/logs">
-                        Logs
-                    </a>
-                    <a class="navbar-item" href="{{ .Session.Me }}">
-                        Profile
-                    </a>
-                </div>
-            {{ end }}
-        </nav>
-
-        <h1 class="title">Ekster - Microsub server</h1>
-
-        {{template "content" .}}
-    </div>
-</section>
-</body>
-</html>
-{{define "header"}}{{end}}
-{{define "content"}}{{end}}
-{{define "footer"}}{{end}}

templates/channel.html

@@ -1,122 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-<meta charset="utf-8">
-<meta name="viewport" content="width=device-width, initial-scale=1">
-<title>Ekster</title>
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.1/css/bulma.min.css">
-</head>
-<body>
-    <section class="section">
-        <div class="container">
-            <nav class="navbar" role="navigation" aria-label="main navigation">
-                <div class="navbar-brand">
-                    <a class="navbar-item" href="/">
-                        Ekster
-                    </a>
-
-                    <a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false" data-target="menu">
-                        <span aria-hidden="true"></span>
-                        <span aria-hidden="true"></span>
-                        <span aria-hidden="true"></span>
-                    </a>
-                </div>
-
-                {{ if .Session.LoggedIn }}
-                    <div id="menu" class="navbar-menu">
-                        <a class="navbar-item" href="/settings">
-                            Settings
-                        </a>
-                        <a class="navbar-item" href="/logs">
-                            Logs
-                        </a>
-                        <a class="navbar-item" href="{{ .Session.Me }}">
-                            Profile
-                        </a>
-                    </div>
-                {{ end }}
-            </nav>
-
-            <h1 class="title">Ekster - Microsub server</h1>
-
-            {{ $channel := .CurrentChannel }}
-
-            <nav class="breadcrumb" aria-label="breadcrumbs">
-                <ul>
-                    <li><a href="/settings">Settings</a></li>
-                    <li class="is-active"><a href="/setttings/channel?uid={{ .CurrentChannel }}">{{ $channel.Name }}</a></li>
-                </ul>
-            </nav>
-
-            <h2 class="subtitle is-2">{{ $channel.Name }}</h2>
-
-            <div class="columns">
-                <div class="column">
-                    <h3 class="title is-4">Settings</h3>
-                    <form action="/settings/channel" method="post">
-                        <input type="hidden" name="uid" value="{{ .CurrentChannel.UID }}" />
-                        <div class="field">
-                            <label class="label" for="exclude_regex">Blocking Regex</label>
-                            <div class="control">
-                                <input type="text" class="input" id="exclude_regex" name="exclude_regex" value="{{ .CurrentSetting.ExcludeRegex }}" placeholder="enter regex to block" />
-                            </div>
-                        </div>
-                        <div class="field">
-                            <label class="label" for="include_regex">Tracking Regex</label>
-                            <div class="control">
-                                <input type="text" class="input" id="include_regex" name="include_regex" value="{{ .CurrentSetting.IncludeRegex }}" placeholder="enter regex to track items" />
-                            </div>
-                        </div>
-                        <div class="field">
-                            <label class="label" for="type">Channel Type</label>
-                            <div class="control">
-                                <div class="select">
-                                    <select name="type" id="type">
-                                        <option value="null" {{if eq (.CurrentSetting.ChannelType) "null" }}selected{{end}}>Null</option>
-                                        <option value="sorted-set" {{if eq (.CurrentSetting.ChannelType) "sorted-set" }}selected{{end}}>Sorted Set</option>
-                                        <option value="stream" {{if eq (.CurrentSetting.ChannelType) "stream" }}selected{{end}}>Streams</option>
-                                        <option value="postgres-stream" {{if eq (.CurrentSetting.ChannelType) "postgres-stream" }}selected{{end}}>Postgres Stream</option>
-                                    </select>
-                                </div>
-                            </div>
-                        </div>
-                        <div class="field">
-                            <label for="exclude_type" class="label">Exclude Types</label>
-                            <div class="control">
-                                <div class="select is-multiple">
-                                    <select name="exclude_type" id="exclude_type" multiple>
-                                        {{ range $key, $excluded := $.ExcludedTypes }}
-                                            <option value="{{ $key }}" {{ if $excluded }}selected="selected"{{ end }}>{{ index $.ExcludedTypeNames $key }}</option>
-                                        {{ end }}
-                                    </select>
-                                </div>
-                            </div>
-                        </div>
-                        <div class="field">
-                            <div class="control">
-                                <button type="submit" class="button is-primary">Save</button>
-                            </div>
-                        </div>
-                    </form>
-                </div>
-
-                <div class="column">
-                    <h3 class="title is-4">Feeds</h3>
-
-                    <div class="channel">
-                        {{ range .Feeds }}
-                            <div class="feed box">
-                                <div class="name">
-                                    <a href="{{ .URL }}">{{ .URL }}</a>
-                                </div>
-                            </div>
-                        {{ else }}
-                            <div class="no-channels">No feeds</div>
-                        {{ end }}
-                    </div>
-                </div>
-            </div>
-        </div>
-    </section>
-</body>
-</html>

templates/index.html

@@ -1,71 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-<meta charset="utf-8">
-<meta name="viewport" content="width=device-width, initial-scale=1">
-<title>Ekster</title>
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.1/css/bulma.min.css">
-<link rel="micropub" href="{{ .Baseurl }}/micropub" />
-<link rel="authorization_endpoint" href="{{ .Baseurl }}/auth" />
-<link rel="token_endpoint" href="{{ .Baseurl }}/auth/token" />
-</head>
-<body>
-    <section class="section">
-        <div class="container">
-
-
-            <nav class="navbar" role="navigation" aria-label="main navigation">
-                <div class="navbar-brand">
-                    <a class="navbar-item" href="/">
-                        Ekster
-                    </a>
-
-                    <a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false" data-target="menu">
-                        <span aria-hidden="true"></span>
-                        <span aria-hidden="true"></span>
-                        <span aria-hidden="true"></span>
-                    </a>
-                </div>
-
-                {{ if .Session.LoggedIn }}
-                    <div id="menu" class="navbar-menu">
-                        <a class="navbar-item" href="/settings">
-                            Settings
-                        </a>
-                        <a class="navbar-item" href="/logs">
-                            Logs
-                        </a>
-                        <a class="navbar-item" href="{{ .Session.Me }}">
-                            Profile
-                        </a>
-                    </div>
-                {{ end }}
-            </nav>
-
-            <h1 class="title">Ekster - Microsub server</h1>
-
-            {{ if .Session.LoggedIn }}
-                <h2 class="title">Logout</h2>
-                <form action="/session/logout" method="post">
-                    <button type="submit" class="button is-info">Logout</button>
-                </form>
-            {{ else }}
-                <h2 class="title">Sign in to Ekster</h2>
-                <form action="/session" method="post">
-                    <div class="field">
-                        <label class="label" for="url"></label>
-                        <div class="control">
-                            <input type="text" name="url" id="url" class="input" placeholder="https://example.com/">
-                        </div>
-                    </div>
-                    <div class="field is-grouped">
-                        <div class="control">
-                            <button type="submit" class="button is-info">Login</button>
-                        </div>
-                    </div>
-                </form>
-            {{ end }}
-        </div>
-    </section>
-</body>
-</html>

templates/logs.html

@@ -1,50 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-<meta charset="utf-8">
-<meta name="viewport" content="width=device-width, initial-scale=1">
-<title>Ekster</title>
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.1/css/bulma.min.css">
-</head>
-<body>
-    <section class="section">
-        <div class="container">
-
-
-            <nav class="navbar" role="navigation" aria-label="main navigation">
-                <div class="navbar-brand">
-                    <a class="navbar-item" href="/">
-                        Ekster
-                    </a>
-
-                    <a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false" data-target="menu">
-                        <span aria-hidden="true"></span>
-                        <span aria-hidden="true"></span>
-                        <span aria-hidden="true"></span>
-                    </a>
-                </div>
-
-                {{ if .Session.LoggedIn }}
-                    <div id="menu" class="navbar-menu">
-                        <a class="navbar-item" href="/settings">
-                            Settings
-                        </a>
-                        <a class="navbar-item" href="/logs">
-                            Logs
-                        </a>
-                        <a class="navbar-item" href="{{ .Session.Me }}">
-                            Profile
-                        </a>
-                    </div>
-                {{ end }}
-            </nav>
-
-            <h1 class="title">Ekster - Microsub server</h1>
-
-            <h2 class="subtitle">Logs</h2>
-
-            <p>Logs</p>
-        </div>
-    </section>
-</body>
-</html>

templates/settings.html

@@ -1,63 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-<meta charset="utf-8">
-<meta name="viewport" content="width=device-width, initial-scale=1">
-<title>Ekster</title>
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.1/css/bulma.min.css">
-</head>
-<body>
-    <section class="section">
-        <div class="container">
-            <nav class="navbar" role="navigation" aria-label="main navigation">
-                <div class="navbar-brand">
-                    <a class="navbar-item" href="/">
-                        Ekster
-                    </a>
-
-                    <a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false" data-target="menu">
-                        <span aria-hidden="true"></span>
-                        <span aria-hidden="true"></span>
-                        <span aria-hidden="true"></span>
-                    </a>
-                </div>
-
-                {{ if .Session.LoggedIn }}
-                    <div id="menu" class="navbar-menu">
-                        <a class="navbar-item" href="/settings">
-                            Settings
-                        </a>
-                        <a class="navbar-item" href="/logs">
-                            Logs
-                        </a>
-                        <a class="navbar-item" href="{{ .Session.Me }}">
-                            Profile
-                        </a>
-                    </div>
-                {{ end }}
-            </nav>
-
-            <h1 class="title">Ekster - Microsub server</h1>
-
-            {{ if .Session.LoggedIn }}
-            {{ end }}
-
-            <h2 class="subtitle">Channels</h2>
-
-            <div class="channels">
-                {{ range .Channels }}
-                    <div class="channel box">
-                        <div class="name">
-                            <a href="/settings/channel?uid={{ .UID }}">
-                                {{ .Name }}
-                            </a>
-                        </div>
-                    </div>
-                {{ else }}
-                    <div class="no-channels">No channels</div>
-                {{ end }}
-            </div>
-        </div>
-    </section>
-</body>
-</html>