Extract fetchAlternateFeed

- Extract findFeeds

.drone.yml

@@ -1,74 +1,25 @@
----
 kind: pipeline
-type: docker
+name: default
-name: build and test
 
 workspace:
   base: /go
   path: src/p83.nl/go/ekster
 
-trigger:
-  event:
-    - push
-
-services:
-  - name: redis
-    image: redis:5
-  - name: database
-    image: postgres:14
-    environment:
-      POSTGRES_DB: ekster_testing
-      POSTGRES_USER: postgres
-      POSTGRES_PASSWORD: simple
-      POSTGRES_HOST_AUTH_METHOD: trust
-
 steps:
   - name: testing
-    image: golang:1.18-alpine
+    image: golang:1.16-alpine
     environment:
       CGO_ENABLED: 0
       GOOS: linux
       GOARCH: amd64
     commands:
-      - go version
       - apk --no-cache add git
       - go get -d -t ./...
-      - go build -buildvcs=false p83.nl/go/ekster/cmd/eksterd
+      - go build p83.nl/go/ekster/cmd/eksterd
-      - go vet ./...
+      - go test ./...
-      - go test -v ./...
-
----
-kind: pipeline
-type: docker
-name: move to production
-
-workspace:
-  base: /go
-  path: src/p83.nl/go/ekster
-
-trigger:
-  event:
-    - promote
-  target:
-    - production
-
-steps:
-  - name: build
-    image: golang:1.18-alpine
-    environment:
-      CGO_ENABLED: 0
-      GOOS: linux
-      GOARCH: amd64
-    commands:
-      - go version
-      - apk --no-cache add git
-      - go get -d -t ./...
-      - go build -buildvcs=false p83.nl/go/ekster/cmd/eksterd
 
   - name: publish-personal
     image: plugins/docker
-    depends_on:
-      - build
     settings:
       repo: registry.stuifzandapp.com/microsub-server
       registry: registry.stuifzandapp.com
@@ -77,10 +28,19 @@ steps:
       password:
         from_secret: docker_password
 
+  - name: publish-docker
+    image: plugins/docker
+    settings:
+      repo: pstuifzand/ekster
+      tags:
+        - alpine
+      username:
+        from_secret: docker_official_username
+      password:
+        from_secret: docker_official_password
+
   - name: deploy
     image: appleboy/drone-ssh
-    depends_on:
-      - publish-personal
     settings:
       host: microsub.stuifzandapp.com
       username: microsub

.github/workflows/go.yml

@@ -1,25 +0,0 @@
-name: Go
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
-
-jobs:
-
-  build:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: 1.16
-
-    - name: Build
-      run: go build -v ./...
-
-    - name: Test
-      run: go test -v ./...

.pre-commit-config.yaml

@@ -7,7 +7,6 @@ repos:
     - id: trailing-whitespace
     - id: end-of-file-fixer
     - id: check-yaml
-      args: [--allow-multiple-documents]
     - id: check-merge-conflict
     - id: check-added-large-files
 - repo: https://github.com/dnephin/pre-commit-golang

CHANGELOG.md

@@ -1,26 +0,0 @@
-# Changelog
-
-## [Unreleased]
-
-## [1.0.0-rc.1] - 2021-11-20
-
-### Added
-
-- Postgresql support for channels, feeds, items and subscriptions.
-- Support "source" in items and feeds
-
-### Changed
-
-- Default channel backend is postgresql
-
-### Fixed
-
-- All `staticcheck` problems are fixed.
-- Dependency between memorybackend and hubbackend removed and simplified.
-
-### Deprecated
-
-- All Redis timeline types are deprecated and will be removed in a later version.
-
-[Unreleased]: https://git.p83.nl/peter/ekster/compare/1.0.0-rc.1...master
-[1.0.0-rc.1]: https://git.p83.nl/peter/ekster/src/tag/1.0.0-rc.1

Dockerfile

@@ -3,4 +3,5 @@ RUN apk --no-cache add ca-certificates
 WORKDIR /opt/micropub
 EXPOSE 80
 COPY ./eksterd /app/
+COPY ./templates /app/templates
 ENTRYPOINT ["/app/eksterd"]

README.md

@@ -140,8 +140,6 @@ support microsub.
       -verbose
             show verbose logging
 
-Instead of the `UID` you can also use the `NAME` of the channel in most functions.
-
 ## Configuration: backend.json
 
 The `backend.json` file contains all information about channels, feeds and authentication.
@@ -155,6 +153,7 @@ The two parts that should be changed are:
     "Me": "...",
     "TokenEndpoint": "...",
 
+
 The `Me` value should be set to the URL you use to sign into Monocle, or
 Micropub client.
 

cmd/ek/main.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 // Ek is a microsub client.
 package main
 
@@ -144,7 +126,7 @@ Usage:
 
 Commands:
 
-	connect URL                  login to a website that supports Indieauth and Microsub
+	connect URL                  login to Indieauth URL, e.g. your website
 
 	channels                     list channels
 	channels NAME                create channel with NAME
@@ -236,26 +218,6 @@ Global arguments:
 	performCommands(&c, flag.Args())
 }
 
-func channelID(sub microsub.Microsub, channelNameOrID string) (string, error) {
-	channels, err := sub.ChannelsGetList()
-	if err != nil {
-		// we encountered an error, so we are not sure if it worked
-		return channelNameOrID, err
-	}
-
-	for _, c := range channels {
-		if c.Name == channelNameOrID {
-			return c.UID, nil
-		}
-		if c.UID == channelNameOrID {
-			return c.UID, nil
-		}
-	}
-
-	// unknown?
-	return channelNameOrID, nil
-}
-
 func performCommands(sub microsub.Microsub, commands []string) {
 	if len(commands) == 0 {
 		flag.Usage()
@@ -283,15 +245,15 @@ func performCommands(sub microsub.Microsub, commands []string) {
 	}
 
 	if len(commands) == 3 && commands[0] == "channels" {
-		if commands[1] == "-delete" {
+		uid := commands[1]
-			uid, _ := channelID(sub, commands[2])
+		if uid == "-delete" {
+			uid = commands[2]
 			err := sub.ChannelsDelete(uid)
 			if err != nil {
 				log.Fatalf("An error occurred: %s\n", err)
 			}
 			fmt.Printf("Channel %s deleted\n", uid)
 		} else {
-			uid, _ := channelID(sub, commands[1])
 			name := commands[2]
 			channel, err := sub.ChannelsUpdate(uid, name)
 			if err != nil {
@@ -302,7 +264,7 @@ func performCommands(sub microsub.Microsub, commands []string) {
 	}
 
 	if len(commands) >= 2 && commands[0] == "timeline" {
-		channel, _ := channelID(sub, commands[1])
+		channel := commands[1]
 
 		var timeline microsub.Timeline
 		var err error
@@ -342,7 +304,7 @@ func performCommands(sub microsub.Microsub, commands []string) {
 		query := commands[1]
 		var channel string
 		if len(commands) == 3 {
-			channel, _ = channelID(sub, commands[2])
+			channel = commands[2]
 		} else {
 			channel = "global"
 		}
@@ -369,7 +331,7 @@ func performCommands(sub microsub.Microsub, commands []string) {
 	}
 
 	if len(commands) == 2 && commands[0] == "follow" {
-		uid, _ := channelID(sub, commands[1])
+		uid := commands[1]
 		feeds, err := sub.FollowGetList(uid)
 		if err != nil {
 			log.Fatalf("An error occurred: %s\n", err)
@@ -380,7 +342,7 @@ func performCommands(sub microsub.Microsub, commands []string) {
 	}
 
 	if len(commands) == 3 && commands[0] == "follow" {
-		uid, _ := channelID(sub, commands[1])
+		uid := commands[1]
 		u := commands[2]
 		_, err := sub.FollowURL(uid, u)
 		if err != nil {
@@ -390,7 +352,7 @@ func performCommands(sub microsub.Microsub, commands []string) {
 	}
 
 	if len(commands) == 3 && commands[0] == "unfollow" {
-		uid, _ := channelID(sub, commands[1])
+		uid := commands[1]
 		u := commands[2]
 		err := sub.UnfollowURL(uid, u)
 		if err != nil {

cmd/eksterd/app.go

@@ -1,109 +0,0 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-package main
-
-import (
-	"fmt"
-	"log"
-	"math/rand"
-	"net/http"
-	"time"
-
-	"github.com/pkg/errors"
-	"p83.nl/go/ekster/pkg/server"
-)
-
-func init() {
-	rand.Seed(time.Now().UnixNano())
-}
-
-// App is the main app structure
-type App struct {
-	options    AppOptions
-	backend    *memoryBackend
-	hubBackend *hubIncomingBackend
-}
-
-// Run runs the app
-func (app *App) Run() error {
-	err := initSearch()
-	if err != nil {
-		return fmt.Errorf("while starting app: %v", err)
-	}
-	app.backend.run()
-	app.hubBackend.run()
-
-	log.Printf("Listening on port %d\n", app.options.Port)
-	return http.ListenAndServe(fmt.Sprintf(":%d", app.options.Port), nil)
-}
-
-// NewApp initializes the App
-func NewApp(options AppOptions) (*App, error) {
-	app := &App{
-		options: options,
-	}
-
-	backend, err := loadMemoryBackend(options.pool, options.database)
-	if err != nil {
-		return nil, err
-	}
-	app.backend = backend
-
-	// FIXME: load from database
-	app.backend.TokenEndpoint = "https://p83.nl/authtoken"
-	app.backend.Me = "https://p83.nl/"
-
-	app.backend.AuthEnabled = options.AuthEnabled
-
-	app.hubBackend = &hubIncomingBackend{
-		baseURL:  options.BaseURL,
-		pool:     options.pool,
-		database: options.database,
-	}
-	app.backend.hubBackend = app.hubBackend
-
-	http.Handle("/micropub", &micropubHandler{
-		Backend: app.backend,
-		pool:    options.pool,
-	})
-
-	handler, broker := server.NewMicrosubHandler(app.backend)
-	if options.AuthEnabled {
-		handler = WithAuth(handler, app.backend)
-	}
-
-	app.backend.broker = broker
-
-	http.Handle("/microsub", handler)
-
-	http.Handle("/incoming/", &incomingHandler{
-		Backend:   app.hubBackend,
-		Processor: app.backend,
-	})
-
-	if !options.Headless {
-		handler, err := newMainHandler(app.backend, options.BaseURL, options.TemplateDir, options.pool)
-		if err != nil {
-			return nil, errors.Wrap(err, "could not create main handler")
-		}
-		http.Handle("/", handler)
-	}
-
-	return app, nil
-}

cmd/eksterd/auth.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package main
 
 import (

cmd/eksterd/database_test.go

@@ -1,123 +0,0 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2022 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-package main
-
-import (
-	"database/sql"
-	"log"
-	"net/http/httptest"
-	"os"
-	"testing"
-
-	"github.com/gomodule/redigo/redis"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/suite"
-)
-
-type DatabaseSuite struct {
-	suite.Suite
-
-	URL      string
-	Database *sql.DB
-
-	RedisURL string
-	Redis    redis.Conn
-}
-
-func (s *DatabaseSuite) SetupSuite() {
-	db, err := sql.Open("postgres", s.URL)
-	if err != nil {
-		log.Fatal(err)
-	}
-	s.Database = db
-
-	conn, err := redis.Dial("tcp", s.RedisURL)
-	if err != nil {
-		log.Fatal(err)
-	}
-	s.Redis = conn
-	_, err = s.Redis.Do("SELECT", "1")
-	if err != nil {
-		log.Fatal(err)
-	}
-}
-
-func (s *DatabaseSuite) TearDownSuite() {
-	err := s.Database.Close()
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	err = s.Redis.Close()
-	if err != nil {
-		log.Fatal(err)
-	}
-}
-
-type databaseSuite struct {
-	DatabaseSuite
-}
-
-func (d *databaseSuite) TestGetChannelFromAuthorization() {
-	_, err := d.Database.Exec(`truncate "sources", "channels", "feeds", "subscriptions","items"`)
-	assert.NoError(d.T(), err, "truncate sources, channels, feeds")
-	row := d.Database.QueryRow(`INSERT INTO "channels" (uid, name, created_at, updated_at) VALUES ('abcdef', 'Channel', now(), now()) RETURNING "id"`)
-	var id int
-	err = row.Scan(&id)
-	assert.NoError(d.T(), err, "insert channel")
-	_, err = d.Database.Exec(`INSERT INTO "sources" (channel_id, auth_code, created_at, updated_at) VALUES ($1, '1234', now(), now())`, id)
-	assert.NoError(d.T(), err, "insert sources")
-
-	// source_id found
-	r := httptest.NewRequest("POST", "/micropub?source_id=1234", nil)
-	_, c, err := getChannelFromAuthorization(r, d.Redis, d.Database)
-	assert.NoError(d.T(), err, "channel from source_id")
-	assert.Equal(d.T(), "abcdef", c, "channel uid found")
-
-	// source_id not found
-	r = httptest.NewRequest("POST", "/micropub?source_id=1111", nil)
-	_, c, err = getChannelFromAuthorization(r, d.Redis, d.Database)
-	assert.Error(d.T(), err, "channel from authorization header")
-	assert.Equal(d.T(), "", c, "channel uid found")
-}
-
-func TestDatabaseSuite(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skip test for database")
-	}
-
-	databaseURL := os.Getenv("DATABASE_TEST_URL")
-	if databaseURL == "" {
-		databaseURL = "host=database user=postgres password=simple dbname=ekster_testing sslmode=disable"
-	}
-	databaseSuite := &databaseSuite{
-		DatabaseSuite{
-			URL:      databaseURL,
-			RedisURL: "redis:6379",
-		},
-	}
-
-	databaseURL = "postgres://postgres@database/ekster_testing?sslmode=disable&user=postgres&password=simple"
-	err := runMigrations(databaseURL)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	suite.Run(t, databaseSuite)
-}

cmd/eksterd/db/migrations/000001_create_channels_table.down.sql

@@ -1,19 +0,0 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-DROP TABLE "channels";

cmd/eksterd/db/migrations/000001_create_channels_table.up.sql

@@ -1,25 +0,0 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-CREATE TABLE IF NOT EXISTS "channels" (
-    "id" int primary key generated always as identity,
-    "uid" varchar(255) unique,
-    "name" varchar(255) unique,
-    "created_at" timestamptz DEFAULT current_timestamp,
-    "updated_at" timestamptz
-);

cmd/eksterd/db/migrations/000002_create_feeds_table.down.sql

@@ -1,19 +0,0 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-DROP TABLE "feeds";

cmd/eksterd/db/migrations/000002_create_feeds_table.up.sql

@@ -1,25 +0,0 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-CREATE TABLE "feeds" (
-     "id" int primary key generated always as identity,
-     "channel_id" int references "channels" (id) on update cascade on delete cascade,
-     "url" varchar(512) not null unique,
-     "created_at" timestamptz DEFAULT current_timestamp,
-     "updated_at" timestamptz
-);

cmd/eksterd/db/migrations/000003_add_default_channels.down.sql

@@ -1,19 +0,0 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-DELETE FROM "channels" WHERE "uid" IN ('home', 'notifications');

cmd/eksterd/db/migrations/000003_add_default_channels.up.sql

@@ -1,19 +0,0 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-INSERT INTO "channels" ("uid", "name") VALUES ('home', 'Home'), ('notifications', 'Notifications');

cmd/eksterd/db/migrations/000004_create_items_table.down.sql

@@ -1,19 +0,0 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-DROP TABLE "items";

cmd/eksterd/db/migrations/000004_create_items_table.up.sql

@@ -1,28 +0,0 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-CREATE TABLE IF NOT EXISTS "items" (
-    "id" int primary key generated always as identity,
-    "channel_id" int references "channels" on delete cascade,
-    "uid" varchar(512) not null unique,
-    "is_read" int default 0,
-    "data" jsonb,
-    "created_at" timestamptz DEFAULT current_timestamp,
-    "updated_at" timestamptz,
-    "published_at" timestamptz
-);

cmd/eksterd/db/migrations/000005_alter_items_add_feed_id.down.sql

@@ -1,19 +0,0 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-ALTER TABLE "items" DROP COLUMN "feed_id";

cmd/eksterd/db/migrations/000005_alter_items_add_feed_id.up.sql

@@ -1,19 +0,0 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-ALTER TABLE "items" ADD COLUMN "feed_id" INT REFERENCES "feeds" ON DELETE CASCADE;

cmd/eksterd/db/migrations/000006_create_subscriptions_table.down.sql

@@ -1,19 +0,0 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-DROP TABLE "subscriptions";

cmd/eksterd/db/migrations/000006_create_subscriptions_table.up.sql

@@ -1,30 +0,0 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-CREATE TABLE "subscriptions" (
-     "id" int primary key generated always as identity,
-     "topic" varchar(1024) not null references "feeds" ("url") on update cascade on delete cascade,
-     "hub" varchar(1024) null,
-     "callback" varchar(1024) null,
-     "subscription_secret" varchar(32) not null,
-     "url_secret" varchar(32) not null,
-     "lease_seconds" int not null,
-     "created_at" timestamptz DEFAULT current_timestamp,
-     "updated_at" timestamptz,
-     "resubscribe_at" timestamptz
-);

cmd/eksterd/db/migrations/000007_alter_table_add_feed_tier.down.sql

@@ -1,4 +0,0 @@
-alter table "feeds"
-    drop column "tier",
-    drop column "unmodified",
-    drop column "next_fetch_at";

cmd/eksterd/db/migrations/000007_alter_table_add_feed_tier.up.sql

@@ -1,4 +0,0 @@
-alter table "feeds"
-    add column "tier" int default 0,
-    add column "unmodified" int default 0,
-    add column "next_fetch_at" timestamptz;

cmd/eksterd/db/migrations/000008_create_table_sources.down.sql

@@ -1 +0,0 @@
-DROP TABLE "sources";

cmd/eksterd/db/migrations/000008_create_table_sources.up.sql

@@ -1,20 +0,0 @@
-BEGIN;
-CREATE OR REPLACE FUNCTION update_timestamp()
-    RETURNS TRIGGER AS $$
-BEGIN
-    NEW.updated_at = now();
-    RETURN NEW;
-END;
-$$ language 'plpgsql';
-COMMIT;
-
-CREATE TABLE "sources" (
-   "id" int primary key generated always as identity,
-   "channel_id" int not null,
-   "auth_code" varchar(64) not null,
-   "created_at" timestamp DEFAULT current_timestamp,
-   "updated_at" timestamp DEFAULT current_timestamp
-);
-
-CREATE TRIGGER sources_update_timestamp BEFORE INSERT OR UPDATE ON "sources"
-FOR EACH ROW EXECUTE PROCEDURE update_timestamp();

cmd/eksterd/feedsearch.go

@@ -0,0 +1,124 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"p83.nl/go/ekster/pkg/fetch"
+	"p83.nl/go/ekster/pkg/microsub"
+	"willnorris.com/go/microformats"
+)
+
+func isSupportedFeedType(feedType string) bool {
+	return strings.HasPrefix(feedType, "text/html") ||
+		strings.HasPrefix(feedType, "application/json") ||
+		strings.HasPrefix(feedType, "application/xml") ||
+		strings.HasPrefix(feedType, "text/xml") ||
+		strings.HasPrefix(feedType, "application/rss+xml") ||
+		strings.HasPrefix(feedType, "application/atom+xml")
+}
+
+func findFeeds(cachingFetch fetch.FetcherFunc, feedURL string) ([]microsub.Feed, error) {
+	resp, err := cachingFetch(feedURL)
+	if err != nil {
+		return nil, fmt.Errorf("while fetching %s: %w", feedURL, err)
+	}
+	defer resp.Body.Close()
+
+	fetchURL, err := url.Parse(feedURL)
+	md := microformats.Parse(resp.Body, fetchURL)
+	if err != nil {
+		return nil, fmt.Errorf("while fetching %s: %w", feedURL, err)
+	}
+
+	feedResp, err := cachingFetch(fetchURL.String())
+	if err != nil {
+		return nil, fmt.Errorf("in fetch of %s: %w", fetchURL, err)
+	}
+	defer feedResp.Body.Close()
+
+	// TODO: Combine FeedHeader and FeedItems so we can use it here
+	parsedFeed, err := fetch.FeedHeader(cachingFetch, fetchURL.String(), feedResp.Header.Get("Content-Type"), feedResp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("in parse of %s: %w", fetchURL, err)
+	}
+
+	var feeds []microsub.Feed
+
+	// TODO: Only include the feed if it contains some items
+	feeds = append(feeds, parsedFeed)
+
+	// Fetch alternates
+	if alts, e := md.Rels["alternate"]; e {
+		for _, alt := range alts {
+			relURL := md.RelURLs[alt]
+			log.Printf("alternate found with type %s %#v\n", relURL.Type, relURL)
+
+			if isSupportedFeedType(relURL.Type) {
+				parsedFeed, err := fetchAlternateFeed(cachingFetch, alt)
+
+				if err != nil {
+					continue
+				}
+
+				feeds = append(feeds, parsedFeed)
+			}
+		}
+	}
+	return feeds, nil
+}
+
+func fetchAlternateFeed(cachingFetch fetch.FetcherFunc, altURL string) (microsub.Feed, error) {
+	feedResp, err := cachingFetch(altURL)
+	if err != nil {
+		return microsub.Feed{}, fmt.Errorf("fetch of %s: %v", altURL, err)
+	}
+
+	defer feedResp.Body.Close()
+
+	parsedFeed, err := fetch.FeedHeader(cachingFetch, altURL, feedResp.Header.Get("Content-Type"), feedResp.Body)
+	if err != nil {
+		return microsub.Feed{}, fmt.Errorf("in parse of %s: %v", altURL, err)
+	}
+
+	return parsedFeed, nil
+}
+
+func getPossibleURLs(query string) []string {
+	urls := []string{}
+	if !(strings.HasPrefix(query, "https://") || strings.HasPrefix(query, "http://")) {
+		secureURL := "https://" + query
+		if checkURL(secureURL) {
+			urls = append(urls, secureURL)
+		} else {
+			unsecureURL := "http://" + query
+			if checkURL(unsecureURL) {
+				urls = append(urls, unsecureURL)
+			}
+		}
+	} else {
+		urls = append(urls, query)
+	}
+	return urls
+}
+
+func checkURL(u string) bool {
+	testURL, err := url.Parse(u)
+	if err != nil {
+		return false
+	}
+
+	resp, err := http.Head(testURL.String())
+
+	if err != nil {
+		log.Printf("Error while HEAD %s: %v\n", u, err)
+		return false
+	}
+
+	defer resp.Body.Close()
+
+	return resp.StatusCode == 200
+}

cmd/eksterd/http.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package main
 
 import (
@@ -124,6 +106,10 @@ func newMainHandler(backend *memoryBackend, baseURL, templateDir string, pool *r
 	return h, nil
 }
 
+func (h *mainHandler) templateFile(filename string) string {
+	return fmt.Sprintf("%s/%s", h.TemplateDir, filename)
+}
+
 func (h *mainHandler) renderTemplate(w io.Writer, filename string, data interface{}) error {
 	fsys, err := fs.Sub(templates, "templates")
 	if err != nil {
@@ -306,7 +292,7 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	err := r.ParseForm()
 	if err != nil {
 		log.Println(err)
-		http.Error(w, fmt.Sprintf("Bad Request: %s", err.Error()), http.StatusBadRequest)
+		http.Error(w, fmt.Sprintf("Bad Request: %s", err.Error()), 400)
 		return
 	}
 
@@ -331,19 +317,15 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		} else if r.URL.Path == "/session/callback" {
 			c, err := r.Cookie("session")
 			if err == http.ErrNoCookie {
-				http.Redirect(w, r, "/", http.StatusFound)
+				http.Redirect(w, r, "/", 302)
 				return
 			} else if err != nil {
-				http.Error(w, "could not read cookie", http.StatusInternalServerError)
+				http.Error(w, "could not read cookie", 500)
 				return
 			}
 
 			sessionVar := c.Value
 			sess, err := loadSession(sessionVar, conn)
-			if err != nil {
-				fmt.Fprintf(w, "ERROR: %q\n", err)
-				return
-			}
 
 			verified, authResponse, err := performIndieauthCallback(h.BaseURL, r, &sess)
 			if err != nil {
@@ -356,9 +338,9 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 				saveSession(sessionVar, &sess, conn)
 				log.Printf("SESSION: %#v\n", sess)
 				if sess.NextURI != "" {
-					http.Redirect(w, r, sess.NextURI, http.StatusFound)
+					http.Redirect(w, r, sess.NextURI, 302)
 				} else {
-					http.Redirect(w, r, "/", http.StatusFound)
+					http.Redirect(w, r, "/", 302)
 				}
 				return
 			}
@@ -366,16 +348,11 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		} else if r.URL.Path == "/settings/channel" {
 			c, err := r.Cookie("session")
 			if err == http.ErrNoCookie {
-				http.Redirect(w, r, "/", http.StatusFound)
+				http.Redirect(w, r, "/", 302)
 				return
 			}
 			sessionVar := c.Value
 			sess, err := loadSession(sessionVar, conn)
-			if err != nil {
-				log.Printf("ERROR: %s\n", err)
-				http.Error(w, "Internal Server Error", http.StatusInternalServerError)
-				return
-			}
 
 			if !isLoggedIn(h.Backend, &sess) {
 				w.WriteHeader(401)
@@ -385,68 +362,66 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 			var page settingsPage
 			page.Session = sess
-			currentChannel := r.URL.Query().Get("uid")
+			currentChannelUID := r.URL.Query().Get("uid")
 			page.Channels, err = h.Backend.ChannelsGetList()
-			if err != nil {
+			page.Feeds, err = h.Backend.FollowGetList(currentChannelUID)
-				log.Printf("ERROR: %s\n", err)
+
-				http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+			var selectedChannel microsub.Channel
-				return
+			found := false
-			}
-			page.Feeds, err = h.Backend.FollowGetList(currentChannel)
-			if err != nil {
-				log.Printf("ERROR: %s\n", err)
-				http.Error(w, "Internal Server Error", http.StatusInternalServerError)
-				return
-			}
 
 			for _, v := range page.Channels {
-				if v.UID == currentChannel {
+				if v.UID == currentChannelUID {
-					page.CurrentChannel = v
+					selectedChannel = v
-					if setting, e := h.Backend.Settings[v.UID]; e {
+					found = true
-						page.CurrentSetting = setting
-					} else {
-						page.CurrentSetting = channelSetting{}
-					}
-					if page.CurrentSetting.ChannelType == "" {
-						page.CurrentSetting.ChannelType = "postgres-stream"
-					}
-					page.ExcludedTypeNames = map[string]string{
-						"repost":   "Reposts",
-						"like":     "Likes",
-						"bookmark": "Bookmarks",
-						"reply":    "Replies",
-						"checkin":  "Checkins",
-					}
-					page.ExcludedTypes = make(map[string]bool)
-					types := []string{"repost", "like", "bookmark", "reply", "checkin"}
-					for _, v := range types {
-						page.ExcludedTypes[v] = false
-					}
-					for _, v := range page.CurrentSetting.ExcludeType {
-						page.ExcludedTypes[v] = true
-					}
 					break
 				}
 			}
 
+			if found {
+				page.CurrentChannel = selectedChannel
+				if setting, e := h.Backend.Settings[selectedChannel.UID]; e {
+					page.CurrentSetting = setting
+				} else {
+					page.CurrentSetting = channelSetting{}
+				}
+				// FIXME: similar code is found in timeline.go
+				if page.CurrentSetting.ChannelType == "" {
+					if selectedChannel.UID == "notifications" {
+						page.CurrentSetting.ChannelType = "stream"
+					} else {
+						page.CurrentSetting.ChannelType = "sorted-set"
+					}
+				}
+				page.ExcludedTypeNames = map[string]string{
+					"repost":   "Reposts",
+					"like":     "Likes",
+					"bookmark": "Bookmarks",
+					"reply":    "Replies",
+					"checkin":  "Checkins",
+				}
+				page.ExcludedTypes = make(map[string]bool)
+				types := []string{"repost", "like", "bookmark", "reply", "checkin"}
+				for _, v := range types {
+					page.ExcludedTypes[v] = false
+				}
+				for _, v := range page.CurrentSetting.ExcludeType {
+					page.ExcludedTypes[v] = true
+				}
+			}
+
 			err = h.renderTemplate(w, "channel.html", page)
 			if err != nil {
-				fmt.Fprintf(w, "ERROR: %s\n", err)
+				http.Error(w, err.Error(), 500)
 			}
 			return
 		} else if r.URL.Path == "/logs" {
 			c, err := r.Cookie("session")
 			if err == http.ErrNoCookie {
-				http.Redirect(w, r, "/", http.StatusFound)
+				http.Redirect(w, r, "/", 302)
 				return
 			}
 			sessionVar := c.Value
 			sess, err := loadSession(sessionVar, conn)
-			if err != nil {
-				log.Printf("ERROR: %s\n", err)
-				http.Error(w, "Internal Server Error", http.StatusInternalServerError)
-				return
-			}
 
 			if !isLoggedIn(h.Backend, &sess) {
 				w.WriteHeader(401)
@@ -465,16 +440,11 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		} else if r.URL.Path == "/settings" {
 			c, err := r.Cookie("session")
 			if err == http.ErrNoCookie {
-				http.Redirect(w, r, "/", http.StatusFound)
+				http.Redirect(w, r, "/", 302)
 				return
 			}
 			sessionVar := c.Value
 			sess, err := loadSession(sessionVar, conn)
-			if err != nil {
-				log.Printf("ERROR: %s\n", err)
-				http.Error(w, "Internal Server Error", http.StatusInternalServerError)
-				return
-			}
 
 			if !isLoggedIn(h.Backend, &sess) {
 				w.WriteHeader(401)
@@ -485,11 +455,6 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			var page settingsPage
 			page.Session = sess
 			page.Channels, err = h.Backend.ChannelsGetList()
-			if err != nil {
-				log.Printf("ERROR: %s\n", err)
-				http.Error(w, "Internal Server Error", http.StatusInternalServerError)
-				return
-			}
 			// page.Feeds = h.Backend.Feeds
 
 			err = h.renderTemplate(w, "settings.html", page)
@@ -504,16 +469,11 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			sessionVar := getSessionCookie(w, r)
 
 			sess, err := loadSession(sessionVar, conn)
-			if err != nil {
-				log.Printf("ERROR: %s\n", err)
-				http.Error(w, "Internal Server Error", http.StatusInternalServerError)
-				return
-			}
 
 			if !isLoggedIn(h.Backend, &sess) {
 				sess.NextURI = r.URL.String()
 				saveSession(sessionVar, &sess, conn)
-				http.Redirect(w, r, "/", http.StatusFound)
+				http.Redirect(w, r, "/", 302)
 				return
 			}
 
@@ -555,11 +515,6 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			page.Scope = scope
 			page.State = state
 			page.Channels, err = h.Backend.ChannelsGetList()
-			if err != nil {
-				log.Printf("ERROR: %s\n", err)
-				http.Error(w, "Internal Server Error", http.StatusInternalServerError)
-				return
-			}
 
 			app, err := getAppInfo(clientID)
 			if err != nil {
@@ -577,7 +532,7 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/session" {
 			c, err := r.Cookie("session")
 			if err == http.ErrNoCookie {
-				http.Redirect(w, r, "/", http.StatusFound)
+				http.Redirect(w, r, "/", 302)
 				return
 			}
 
@@ -588,7 +543,7 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 			endpoints, err := getEndpoints(me)
 			if err != nil {
-				http.Error(w, fmt.Sprintf("Bad Request: %s, %s", err.Error(), me), http.StatusBadRequest)
+				http.Error(w, fmt.Sprintf("Bad Request: %s, %s", err.Error(), me), 400)
 				return
 			}
 
@@ -598,7 +553,7 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			sess, err := loadSession(sessionVar, conn)
 
 			if err != nil {
-				http.Redirect(w, r, "/", http.StatusFound)
+				http.Redirect(w, r, "/", 302)
 				return
 			}
 
@@ -610,12 +565,12 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 			err = saveSession(sessionVar, &sess, conn)
 			if err != nil {
-				http.Redirect(w, r, "/", http.StatusFound)
+				http.Redirect(w, r, "/", 302)
 				return
 			}
 
 			authenticationURL := indieauth.CreateAuthenticationURL(*endpoints.AuthorizationEndpoint, endpoints.Me.String(), h.BaseURL, redirectURI, state)
-			http.Redirect(w, r, authenticationURL, http.StatusFound)
+			http.Redirect(w, r, authenticationURL, 302)
 
 			return
 		} else if r.URL.Path == "/session/logout" {
@@ -668,7 +623,7 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			redirectURI.RawQuery = q.Encode()
 
 			log.Println(redirectURI)
-			http.Redirect(w, r, redirectURI.String(), http.StatusFound)
+			http.Redirect(w, r, redirectURI.String(), 302)
 			return
 		} else if r.URL.Path == "/auth/token" {
 			grantType := r.FormValue("grant_type")
@@ -711,40 +666,45 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			}
 
 			w.Header().Add("Content-Type", "application/json")
-			if err := json.NewEncoder(w).Encode(&res); err != nil {
+			enc := json.NewEncoder(w)
+			err = enc.Encode(&res)
+			if err != nil {
 				log.Println(err)
+				fmt.Fprintf(w, "ERROR: %q", err)
 				return
 			}
 			return
 		} else if r.URL.Path == "/settings/channel" {
-			// defer h.Backend.save()
+			defer h.Backend.save()
-			// uid := r.FormValue("uid")
+			uid := r.FormValue("uid")
-			//
-			// if h.Backend.Settings == nil {
-			// 	h.Backend.Settings = make(map[string]channelSetting)
-			// }
-			//
-			// excludeRegex := r.FormValue("exclude_regex")
-			// includeRegex := r.FormValue("include_regex")
-			// channelType := r.FormValue("type")
-			//
-			// setting, e := h.Backend.Settings[uid]
-			// if !e {
-			// 	setting = channelSetting{}
-			// }
-			// setting.ExcludeRegex = excludeRegex
-			// setting.IncludeRegex = includeRegex
-			// setting.ChannelType = channelType
-			// if values, e := r.Form["exclude_type"]; e {
-			// 	setting.ExcludeType = values
-			// }
-			// h.Backend.Settings[uid] = setting
 
-			http.Redirect(w, r, "/settings", http.StatusFound)
+			if h.Backend.Settings == nil {
+				h.Backend.Settings = make(map[string]channelSetting)
+			}
+
+			excludeRegex := r.FormValue("exclude_regex")
+			includeRegex := r.FormValue("include_regex")
+			channelType := r.FormValue("type")
+
+			setting, e := h.Backend.Settings[uid]
+			if !e {
+				setting = channelSetting{}
+			}
+			setting.ExcludeRegex = excludeRegex
+			setting.IncludeRegex = includeRegex
+			setting.ChannelType = channelType
+			if values, e := r.Form["exclude_type"]; e {
+				setting.ExcludeType = values
+			}
+			h.Backend.Settings[uid] = setting
+
+			h.Backend.Debug()
+
+			http.Redirect(w, r, "/settings", 302)
 			return
 		} else if r.URL.Path == "/refresh" {
 			h.Backend.RefreshFeeds()
-			http.Redirect(w, r, "/", http.StatusFound)
+			http.Redirect(w, r, "/", 302)
 			return
 		}
 	}
@@ -755,14 +715,14 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 func httpSessionLogout(r *http.Request, w http.ResponseWriter, conn redis.Conn) {
 	c, err := r.Cookie("session")
 	if err == http.ErrNoCookie {
-		http.Redirect(w, r, "/", http.StatusFound)
+		http.Redirect(w, r, "/", 302)
 		return
 	}
 	if err == nil {
 		sessionVar := c.Value
 		_, _ = conn.Do("DEL", "session:"+sessionVar)
 	}
-	http.Redirect(w, r, "/", http.StatusFound)
+	http.Redirect(w, r, "/", 302)
 }
 
 type parsedEndpoints struct {

cmd/eksterd/hubbackend.go

@@ -1,32 +1,17 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package main
 
 import (
-	"database/sql"
 	"expvar"
 	"fmt"
 	"io"
 	"log"
 	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
 	"time"
 
+	"github.com/pkg/errors"
 	"p83.nl/go/ekster/pkg/util"
 	"p83.nl/go/ekster/pkg/websub"
 
@@ -38,29 +23,31 @@ const LeaseSeconds = 24 * 60 * 60
 
 // HubBackend handles information for the incoming handler
 type HubBackend interface {
+	GetFeeds() []Feed // Deprecated
 	Feeds() ([]Feed, error)
-	CreateFeed(url string) (int64, error)
+	CreateFeed(url, channel string) (int64, error)
 	GetSecret(feedID int64) string
-	UpdateFeed(processor ContentProcessor, feedID int64, contentType string, body io.Reader) error
+	UpdateFeed(feedID int64, contentType string, body io.Reader) error
 	FeedSetLeaseSeconds(feedID int64, leaseSeconds int64) error
 	Subscribe(feed *Feed) error
 }
 
 type hubIncomingBackend struct {
-	baseURL  string
+	backend *memoryBackend
-	pool     *redis.Pool
+	baseURL string
-	database *sql.DB
+	pool    *redis.Pool
 }
 
 // Feed contains information about the feed subscriptions
 type Feed struct {
-	ID            int64
+	ID            int64  `redis:"id"`
-	URL           string
+	Channel       string `redis:"channel"`
-	Callback      string
+	URL           string `redis:"url"`
-	Hub           string
+	Callback      string `redis:"callback"`
-	Secret        string
+	Hub           string `redis:"hub"`
-	LeaseSeconds  int64
+	Secret        string `redis:"secret"`
-	ResubscribeAt *time.Time
+	LeaseSeconds  int64  `redis:"lease_seconds"`
+	ResubscribeAt int64  `redis:"resubscribe_at"`
 }
 
 var (
@@ -72,35 +59,29 @@ func init() {
 }
 
 func (h *hubIncomingBackend) GetSecret(id int64) string {
-	db := h.database
+	conn := h.pool.Get()
-	var secret string
+	defer conn.Close()
-	err := db.QueryRow(
+	secret, err := redis.String(conn.Do("HGET", fmt.Sprintf("feed:%d", id), "secret"))
-		`select "subscription_secret" from "subscriptions" where "id" = $1`,
-		id,
-	).Scan(&secret)
 	if err != nil {
 		return ""
 	}
 	return secret
 }
 
-func (h *hubIncomingBackend) CreateFeed(topic string) (int64, error) {
+func (h *hubIncomingBackend) CreateFeed(topic string, channel string) (int64, error) {
-	log.Println("CreateFeed", topic)
+	conn := h.pool.Get()
-	db := h.database
+	defer conn.Close()
 
-	secret := util.RandStringBytes(32)
+	// TODO(peter): check if topic already is registered
-	urlSecret := util.RandStringBytes(32)
+	id, err := redis.Int64(conn.Do("INCR", "feed:next_id"))
-
-	var subscriptionID int
-	err := db.QueryRow(`
-INSERT INTO "subscriptions" ("topic","subscription_secret", "url_secret", "lease_seconds", "created_at")
-VALUES ($1, $2, $3, $4, DEFAULT) RETURNING "id"`, topic, secret, urlSecret, 60*60*24*7).Scan(&subscriptionID)
 	if err != nil {
 		return 0, err
 	}
-	if err != nil {
+
-		return 0, fmt.Errorf("insert into subscriptions: %w", err)
+	conn.Do("HSET", fmt.Sprintf("feed:%d", id), "url", topic)
-	}
+	conn.Do("HSET", fmt.Sprintf("feed:%d", id), "channel", channel)
+	secret := util.RandStringBytes(16)
+	conn.Do("HSET", fmt.Sprintf("feed:%d", id), "secret", secret)
 
 	client := &http.Client{}
 
@@ -110,107 +91,135 @@ VALUES ($1, $2, $3, $4, DEFAULT) RETURNING "id"`, topic, secret, urlSecret, 60*6
 		return 0, err
 	}
 
-	callbackURL := fmt.Sprintf("%s/incoming/%d", h.baseURL, subscriptionID)
+	callbackURL := fmt.Sprintf("%s/incoming/%d", h.baseURL, id)
 
 	log.Printf("WebSub Hub URL found for topic=%q hub=%q callback=%q\n", topic, hubURL, callbackURL)
 
 	if err == nil && hubURL != "" {
-		_, err := db.Exec(`UPDATE subscriptions SET hub = $1, callback = $2 WHERE id = $3`, hubURL, callbackURL, subscriptionID)
+		args := redis.Args{}.Add(fmt.Sprintf("feed:%d", id), "hub", hubURL, "callback", callbackURL)
+		_, err = conn.Do("HMSET", args...)
 		if err != nil {
-			return 0, fmt.Errorf("save hub and callback: %w", err)
+			return 0, errors.Wrap(err, "could not write to redis backend")
 		}
 	} else {
-		return int64(subscriptionID), nil
+		return id, nil
 	}
 
 	err = websub.Subscribe(client, hubURL, topic, callbackURL, secret, 24*3600)
 	if err != nil {
-		return 0, fmt.Errorf("subscribe: %w", err)
+		return 0, err
 	}
 
-	return int64(subscriptionID), nil
+	return id, nil
 }
 
-func (h *hubIncomingBackend) UpdateFeed(processor ContentProcessor, subscriptionID int64, contentType string, body io.Reader) error {
+func (h *hubIncomingBackend) UpdateFeed(feedID int64, contentType string, body io.Reader) error {
-	log.Println("UpdateFeed", subscriptionID)
+	conn := h.pool.Get()
-
+	defer conn.Close()
-	db := h.database
+	log.Printf("updating feed %d", feedID)
-	// Process all channels that contains this feed
+	u, err := redis.String(conn.Do("HGET", fmt.Sprintf("feed:%d", feedID), "url"))
-	rows, err := db.Query(`
+	if err != nil {
-select topic, c.uid, f.id, c.name
+		return err
-from subscriptions s
+	}
-inner join feeds f    on f.url = s.topic
+	channel, err := redis.String(conn.Do("HGET", fmt.Sprintf("feed:%d", feedID), "channel"))
-inner join channels c on c.id = f.channel_id
-where s.id = $1
-`,
-		subscriptionID,
-	)
 	if err != nil {
 		return err
 	}
 
-	for rows.Next() {
+	log.Printf("Updating feed %d - %s %s\n", feedID, u, channel)
-		var topic, channel, feedID, channelName string
+	err = h.backend.ProcessContent(channel, u, contentType, body)
-
+	if err != nil {
-		err = rows.Scan(&topic, &channel, &feedID, &channelName)
+		log.Printf("could not process content for channel %s: %s", channel, err)
-		if err != nil {
-			log.Println(err)
-			continue
-		}
-
-		log.Printf("Updating feed %s %q in %q (%s)\n", feedID, topic, channelName, channel)
-		_, err = processor.ProcessContent(channel, feedID, topic, contentType, body)
-		if err != nil {
-			log.Printf("could not process content for channel %s: %s", channelName, err)
-		}
 	}
 
 	return err
 }
 
-func (h *hubIncomingBackend) FeedSetLeaseSeconds(subscriptionID int64, leaseSeconds int64) error {
+func (h *hubIncomingBackend) FeedSetLeaseSeconds(feedID int64, leaseSeconds int64) error {
-	db := h.database
+	conn := h.pool.Get()
-	_, err := db.Exec(`
+	defer conn.Close()
-update subscriptions
+	log.Printf("updating feed %d lease_seconds", feedID)
-set lease_seconds = $1,
+
-    resubscribe_at = now() + $2 * interval '1' second
+	args := redis.Args{}.Add(fmt.Sprintf("feed:%d", feedID), "lease_seconds", leaseSeconds, "resubscribe_at", time.Now().Add(time.Duration(60*(leaseSeconds-15))*time.Second).Unix())
-where id = $3
+	_, err := conn.Do("HMSET", args...)
-`, leaseSeconds, leaseSeconds, subscriptionID)
+	if err != nil {
-	return err
+		log.Println(err)
+		return err
+	}
+
+	return nil
+}
+
+// GetFeeds is deprecated, use Feeds instead
+func (h *hubIncomingBackend) GetFeeds() []Feed {
+	log.Println("GetFeeds called, consider replacing with Feeds")
+	feeds, err := h.Feeds()
+	if err != nil {
+		log.Printf("Feeds returned an error: %v", err)
+	}
+	return feeds
 }
 
 // Feeds returns a list of subscribed feeds
 func (h *hubIncomingBackend) Feeds() ([]Feed, error) {
-	db := h.database
+	conn := h.pool.Get()
-	var feeds []Feed
+	defer conn.Close()
+	feeds := []Feed{}
 
-	rows, err := db.Query(`
+	// FIXME(peter): replace with set of currently checked feeds
-		select s.id, topic, hub, callback, subscription_secret, lease_seconds, resubscribe_at
+	feedKeys, err := redis.Strings(conn.Do("KEYS", "feed:*"))
-		from subscriptions s
-		inner join feeds f on f.url = s.topic
-		inner join channels c on c.id = f.channel_id
-		where hub is not null
-	`)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "could not get feeds from backend")
 	}
 
-	for rows.Next() {
+	for _, feedKey := range feedKeys {
 		var feed Feed
-
+		values, err := redis.Values(conn.Do("HGETALL", feedKey))
-		err = rows.Scan(
-			&feed.ID,
-			&feed.URL,
-			&feed.Hub,
-			&feed.Callback,
-			&feed.Secret,
-			&feed.LeaseSeconds,
-			&feed.ResubscribeAt,
-		)
 		if err != nil {
-			log.Println("Feeds: scan subscriptions:", err)
+			log.Printf("could not get feed info for key %s: %v", feedKey, err)
 			continue
 		}
+
+		err = redis.ScanStruct(values, &feed)
+		if err != nil {
+			log.Printf("could not scan struct for key %s: %v", feedKey, err)
+			continue
+		}
+
+		// Add feed id
+		if feed.ID == 0 {
+			parts := strings.Split(feedKey, ":")
+			if len(parts) == 2 {
+				feed.ID, _ = strconv.ParseInt(parts[1], 10, 64)
+				_, err = conn.Do("HSET", feedKey, "id", feed.ID)
+				if err != nil {
+					log.Printf("could not save id for %s: %v", feedKey, err)
+				}
+			}
+		}
+
+		// Fix the callback url
+		callbackURL, err := url.Parse(feed.Callback)
+		if err != nil || !callbackURL.IsAbs() {
+			if err != nil {
+				log.Printf("could not parse callback url %q: %v", callbackURL, err)
+			} else {
+				log.Printf("url is relative; replace with absolute url: %q", callbackURL)
+			}
+
+			feed.Callback = fmt.Sprintf("%s/incoming/%d", h.baseURL, feed.ID)
+			_, err = conn.Do("HSET", feedKey, "callback", feed.Callback)
+			if err != nil {
+				log.Printf("could not save id for %s: %v", feedKey, err)
+			}
+		}
+
+		// Skip feeds without a Hub
+		if feed.Hub == "" {
+			continue
+		}
+
+		log.Printf("Websub feed: %#v\n", feed)
 		feeds = append(feeds, feed)
 	}
 
@@ -218,33 +227,28 @@ func (h *hubIncomingBackend) Feeds() ([]Feed, error) {
 }
 
 func (h *hubIncomingBackend) Subscribe(feed *Feed) error {
-	log.Println("Subscribe", feed.URL)
 	client := http.Client{}
 	return websub.Subscribe(&client, feed.Hub, feed.URL, feed.Callback, feed.Secret, LeaseSeconds)
 }
 
 func (h *hubIncomingBackend) run() error {
-	ticker := time.NewTicker(1 * time.Minute)
+	ticker := time.NewTicker(10 * time.Minute)
 	quit := make(chan struct{})
 
 	go func() {
 		for {
 			select {
 			case <-ticker.C:
-				log.Println("Getting feeds for WebSub started")
+				log.Println("Getting feeds for WebSub")
 				varWebsub.Add("runs", 1)
 
 				feeds, err := h.Feeds()
 				if err != nil {
-					log.Println("Feeds failed:", err)
-					log.Println("Getting feeds for WebSub completed")
-					continue
 				}
 
-				log.Printf("Found %d feeds", len(feeds))
 				for _, feed := range feeds {
 					log.Printf("Looking at %s\n", feed.URL)
-					if feed.ResubscribeAt != nil && time.Now().After(*feed.ResubscribeAt) {
+					if feed.ResubscribeAt == 0 || time.Now().After(time.Unix(feed.ResubscribeAt, 0)) {
 						if feed.Callback == "" {
 							feed.Callback = fmt.Sprintf("%s/incoming/%d", h.baseURL, feed.ID)
 						}
@@ -257,8 +261,6 @@ func (h *hubIncomingBackend) run() error {
 						}
 					}
 				}
-
-				log.Println("Getting feeds for WebSub completed")
 			case <-quit:
 				ticker.Stop()
 				return

cmd/eksterd/incoming.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package main
 
 import (
@@ -31,8 +13,7 @@ import (
 )
 
 type incomingHandler struct {
-	Backend   HubBackend
+	Backend HubBackend
-	Processor ContentProcessor
 }
 
 var (
@@ -68,14 +49,12 @@ func (h *incomingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 			leaseSeconds, err := strconv.ParseInt(leaseStr, 10, 64)
 			if err != nil {
-				log.Printf("error in hub.lease_seconds format %q: %s", leaseSeconds, err)
+				http.Error(w, fmt.Sprintf("error in hub.lease_seconds format %q: %s", leaseSeconds, err), 400)
-				http.Error(w, fmt.Sprintf("error in hub.lease_seconds format %q: %s", leaseSeconds, err), http.StatusBadRequest)
 				return
 			}
 			err = h.Backend.FeedSetLeaseSeconds(feed, leaseSeconds)
 			if err != nil {
-				log.Printf("error in while setting hub.lease_seconds: %s", err)
+				http.Error(w, fmt.Sprintf("error in while setting hub.lease_seconds: %s", err), 400)
-				http.Error(w, fmt.Sprintf("error in while setting hub.lease_seconds: %s", err), http.StatusBadRequest)
 				return
 			}
 		}
@@ -88,7 +67,7 @@ func (h *incomingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if r.Method != http.MethodPost {
-		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+		http.Error(w, "Method not allowed", 405)
 		return
 	}
 
@@ -96,31 +75,28 @@ func (h *incomingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	secret := h.Backend.GetSecret(feed)
 	if secret == "" {
 		log.Printf("missing secret for feed %d\n", feed)
-		http.Error(w, "Unknown", http.StatusBadRequest)
+		http.Error(w, "Unknown", 400)
 		return
 	}
 
 	feedContent, err := ioutil.ReadAll(r.Body)
-	if err != nil {
-		log.Printf("ERROR: %s\n", err)
-		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
-		return
-	}
 
 	// match signature
 	sig := r.Header.Get("X-Hub-Signature")
 	if sig != "" {
 		if err := websub.ValidateHubSignature(sig, feedContent, []byte(secret)); err != nil {
 			log.Printf("could not validate signature: %+v", err)
-			http.Error(w, fmt.Sprintf("could not validate signature: %s", err), http.StatusBadRequest)
+			http.Error(w, fmt.Sprintf("could not validate signature: %s", err), 400)
 			return
 		}
 	}
 
 	ct := r.Header.Get("Content-Type")
-	err = h.Backend.UpdateFeed(h.Processor, feed, ct, bytes.NewBuffer(feedContent))
+	err = h.Backend.UpdateFeed(feed, ct, bytes.NewBuffer(feedContent))
 	if err != nil {
-		http.Error(w, fmt.Sprintf("could not update feed: %s (%s)", ct, err), http.StatusBadRequest)
+		http.Error(w, fmt.Sprintf("could not update feed: %s (%s)", ct, err), 400)
 		return
 	}
+
+	return
 }

cmd/eksterd/main.go

@@ -1,20 +1,14 @@
-/*
+// Copyright (C) 2018 Peter Stuifzand
- *  Ekster is a microsub server
+//
- *  Copyright (c) 2021 The Ekster authors
+// This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public
- *
+// License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any
- * This program is free software: you can redistribute it and/or modify
+// later version.
- * it under the terms of the GNU General Public License as published by
+//
- * the Free Software Foundation, either version 3 of the License, or
+// This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
- * (at your option) any later version.
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
- *
+//
- * This program is distributed in the hope that it will be useful,
+// You should have received a copy of the GNU General Public License along with this program. If not,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
+// see <http://www.gnu.org/licenses/>.
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
 
 /*
 Eksterd is a microsub server that is extendable.
@@ -23,20 +17,18 @@ package main
 
 import (
 	"database/sql"
-	"embed"
-	_ "expvar"
 	"flag"
+	"fmt"
 	"log"
 	"net/http"
 	"os"
 	"time"
 
+	"github.com/gomodule/redigo/redis"
+	"github.com/pkg/errors"
 	"p83.nl/go/ekster/pkg/auth"
 
-	"github.com/golang-migrate/migrate/v4"
+	"p83.nl/go/ekster/pkg/server"
-	_ "github.com/golang-migrate/migrate/v4/database/postgres"
-	"github.com/golang-migrate/migrate/v4/source/iofs"
-	"github.com/gomodule/redigo/redis"
 )
 
 // AppOptions are options for the app
@@ -51,9 +43,6 @@ type AppOptions struct {
 	database    *sql.DB
 }
 
-//go:embed db/migrations/*.sql
-var migrations embed.FS
-
 func init() {
 	log.SetFlags(log.Lshortfile | log.Ldate | log.Ltime)
 }
@@ -92,13 +81,13 @@ func WithAuth(handler http.Handler, b *memoryBackend) http.Handler {
 		}
 		if !authorized {
 			log.Printf("Token could not be validated")
-			http.Error(w, "Can't validate token", http.StatusForbidden)
+			http.Error(w, "Can't validate token", 403)
 			return
 		}
 
 		if token.Me != b.Me { // FIXME: Me should be part of the request
 			log.Printf("Missing \"me\" in token response: %#v\n", token)
-			http.Error(w, "Wrong me", http.StatusForbidden)
+			http.Error(w, "Wrong me", 403)
 			return
 		}
 
@@ -106,6 +95,73 @@ func WithAuth(handler http.Handler, b *memoryBackend) http.Handler {
 	})
 }
 
+// App is the main app structure
+type App struct {
+	options    AppOptions
+	backend    *memoryBackend
+	hubBackend *hubIncomingBackend
+}
+
+// Run runs the app
+func (app *App) Run() error {
+	err := initSearch()
+	if err != nil {
+		return fmt.Errorf("while starting app: %v", err)
+	}
+	app.backend.run()
+	app.hubBackend.run()
+
+	log.Printf("Listening on port %d\n", app.options.Port)
+	return http.ListenAndServe(fmt.Sprintf(":%d", app.options.Port), nil)
+}
+
+// NewApp initializes the App
+func NewApp(options AppOptions) (*App, error) {
+	app := &App{
+		options: options,
+	}
+
+	backend, err := loadMemoryBackend(options.pool, options.database)
+	if err != nil {
+		return nil, err
+	}
+	app.backend = backend
+	app.backend.AuthEnabled = options.AuthEnabled
+	app.backend.baseURL = options.BaseURL
+	app.backend.hubIncomingBackend.pool = options.pool
+	app.backend.hubIncomingBackend.baseURL = options.BaseURL
+
+	app.hubBackend = &hubIncomingBackend{backend: app.backend, baseURL: options.BaseURL, pool: options.pool}
+
+	http.Handle("/micropub", &micropubHandler{
+		Backend: app.backend,
+		pool:    options.pool,
+	})
+
+	handler, broker := server.NewMicrosubHandler(app.backend)
+	if options.AuthEnabled {
+		handler = WithAuth(handler, app.backend)
+	}
+
+	app.backend.broker = broker
+
+	http.Handle("/microsub", handler)
+
+	http.Handle("/incoming/", &incomingHandler{
+		Backend: app.hubBackend,
+	})
+
+	if !options.Headless {
+		handler, err := newMainHandler(app.backend, options.BaseURL, options.TemplateDir, options.pool)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not create main handler")
+		}
+		http.Handle("/", handler)
+	}
+
+	return app, nil
+}
+
 func main() {
 	log.Println("eksterd - microsub server")
 
@@ -141,36 +197,28 @@ func main() {
 			log.Fatal("EKSTER_TEMPLATES environment variable not found, use env var or -templates dir option")
 		}
 	}
-	//
-	// createBackend := false
-	// args := flag.Args()
-	//
-	// if len(args) >= 1 {
-	// 	if args[0] == "new" {
-	// 		createBackend = true
-	// 	}
-	// }
-	//
-	// if createBackend {
-	// 	err := createMemoryBackend()
-	// 	if err != nil {
-	// 		log.Fatalf("Error while saving backend.json: %s", err)
-	// 	}
-	//
-	// TODO(peter): automatically gather this information from login or otherwise
-	//
-	// 	log.Println(`Config file "backend.json" is created in the current directory.`)
-	// 	log.Println(`Update "Me" variable to your website address "https://example.com/"`)
-	// 	log.Println(`Update "TokenEndpoint" variable to the address of your token endpoint "https://example.com/token"`)
-	//
-	// 	return
-	// }
 
-	// TODO(peter): automatically gather this information from login or otherwise
+	createBackend := false
-	databaseURL := "postgres://postgres@database/ekster?sslmode=disable&user=postgres&password=simple"
+	args := flag.Args()
-	err := runMigrations(databaseURL)
+
-	if err != nil {
+	if len(args) >= 1 {
-		log.Fatalf("Error with migrations: %s", err)
+		if args[0] == "new" {
+			createBackend = true
+		}
+	}
+
+	if createBackend {
+		err := createMemoryBackend()
+		if err != nil {
+			log.Fatalf("Error while saving backend.json: %s", err)
+		}
+
+		// TODO(peter): automatically gather this information from login or otherwise
+		log.Println(`Config file "backend.json" is created in the current directory.`)
+		log.Println(`Update "Me" variable to your website address "https://example.com/"`)
+		log.Println(`Update "TokenEndpoint" variable to the address of your token endpoint "https://example.com/token"`)
+
+		return
 	}
 
 	pool := newPool(options.RedisServer)
@@ -190,38 +238,3 @@ func main() {
 
 	db.Close()
 }
-
-// Log migrations
-type Log struct {
-}
-
-// Printf for migrations logs
-func (l Log) Printf(format string, v ...interface{}) {
-	log.Printf(format, v...)
-}
-
-// Verbose returns false
-func (l Log) Verbose() bool {
-	return false
-}
-
-func runMigrations(databaseURL string) error {
-	d, err := iofs.New(migrations, "db/migrations")
-	if err != nil {
-		return err
-	}
-	m, err := migrate.NewWithSourceInstance("iofs", d, databaseURL)
-	if err != nil {
-		return err
-	}
-	defer m.Close()
-	m.Log = &Log{}
-	log.Println("Running migrations")
-	if err = m.Up(); err != nil {
-		if err != migrate.ErrNoChange {
-			return err
-		}
-	}
-	log.Println("Migrations are up")
-	return nil
-}

cmd/eksterd/memory_test.go

@@ -1,228 +1,114 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package main
 
-// func Test_memoryBackend_ChannelsCreate(t *testing.T) {
+import (
-// 	type fields struct {
+	"reflect"
-// 		hubIncomingBackend hubIncomingBackend
+	"sync"
-// 		lock               sync.RWMutex
+	"testing"
-// 		Channels           map[string]microsub.Channel
+	"time"
-// 		Feeds              map[string][]microsub.Feed
+
-// 		Settings           map[string]channelSetting
+	"github.com/gomodule/redigo/redis"
-// 		NextUID            int
+	"p83.nl/go/ekster/pkg/microsub"
-// 		Me                 string
+	"p83.nl/go/ekster/pkg/sse"
-// 		TokenEndpoint      string
+)
-// 		AuthEnabled        bool
+
-// 		ticker             *time.Ticker
+func Test_memoryBackend_ChannelsCreate(t *testing.T) {
-// 		quit               chan struct{}
+	type fields struct {
-// 		broker             *sse.Broker
+		hubIncomingBackend hubIncomingBackend
-// 		pool               *redis.Pool
+		lock               sync.RWMutex
-// 	}
+		Channels           map[string]microsub.Channel
-// 	type args struct {
+		Feeds              map[string][]microsub.Feed
-// 		name string
+		Settings           map[string]channelSetting
-// 	}
+		NextUID            int
-// 	tests := []struct {
+		Me                 string
-// 		name    string
+		TokenEndpoint      string
-// 		fields  fields
+		AuthEnabled        bool
-// 		args    args
+		ticker             *time.Ticker
-// 		want    microsub.Channel
+		quit               chan struct{}
-// 		wantErr bool
+		broker             *sse.Broker
-// 	}{
+		pool               *redis.Pool
-// 		{
+	}
-// 			name: "Duplicate channel",
+	type args struct {
-// 			fields: fields{
+		name string
-// 				hubIncomingBackend: hubIncomingBackend{},
+	}
-// 				lock:               sync.RWMutex{},
+	tests := []struct {
-// 				Channels: func() map[string]microsub.Channel {
+		name    string
-// 					channels := make(map[string]microsub.Channel)
+		fields  fields
-// 					channels["1234"] = microsub.Channel{
+		args    args
-// 						UID:  "1234",
+		want    microsub.Channel
-// 						Name: "Test",
+		wantErr bool
-// 						Unread: microsub.Unread{
+	}{
-// 							Type:        microsub.UnreadCount,
+		{
-// 							Unread:      false,
+			name: "Duplicate channel",
-// 							UnreadCount: 0,
+			fields: fields{
-// 						},
+				hubIncomingBackend: hubIncomingBackend{},
-// 					}
+				lock:               sync.RWMutex{},
-// 					return channels
+				Channels: func() map[string]microsub.Channel {
-// 				}(),
+					channels := make(map[string]microsub.Channel)
-// 				Feeds: func() map[string][]microsub.Feed {
+					channels["1234"] = microsub.Channel{
-// 					feeds := make(map[string][]microsub.Feed)
+						UID:  "1234",
-// 					return feeds
+						Name: "Test",
-// 				}(),
+						Unread: microsub.Unread{
-// 				Settings:      nil,
+							Type:        microsub.UnreadCount,
-// 				NextUID:       1,
+							Unread:      false,
-// 				Me:            "",
+							UnreadCount: 0,
-// 				TokenEndpoint: "",
+						},
-// 				AuthEnabled:   false,
+					}
-// 				ticker:        nil,
+					return channels
-// 				quit:          nil,
+				}(),
-// 				broker:        nil,
+				Feeds: func() map[string][]microsub.Feed {
-// 				pool:          nil,
+					feeds := make(map[string][]microsub.Feed)
-// 			},
+					return feeds
-// 			args: args{
+				}(),
-// 				name: "Test",
+				Settings:      nil,
-// 			},
+				NextUID:       1,
-// 			want: microsub.Channel{
+				Me:            "",
-// 				UID:  "1234",
+				TokenEndpoint: "",
-// 				Name: "Test",
+				AuthEnabled:   false,
-// 				Unread: microsub.Unread{
+				ticker:        nil,
-// 					Type:        microsub.UnreadCount,
+				quit:          nil,
-// 					Unread:      false,
+				broker:        nil,
-// 					UnreadCount: 0,
+				pool:          nil,
-// 				},
+			},
-// 			},
+			args: args{
-// 			wantErr: false,
+				name: "Test",
-// 		},
+			},
-// 	}
+			want: microsub.Channel{
-// 	for _, tt := range tests {
+				UID:  "1234",
-// 		t.Run(tt.name, func(t *testing.T) {
+				Name: "Test",
-// 			b := &memoryBackend{
+				Unread: microsub.Unread{
-// 				hubIncomingBackend: tt.fields.hubIncomingBackend,
+					Type:        microsub.UnreadCount,
-// 				lock:               tt.fields.lock,
+					Unread:      false,
-// 				Channels:           tt.fields.Channels,
+					UnreadCount: 0,
-// 				Feeds:              tt.fields.Feeds,
+				},
-// 				Settings:           tt.fields.Settings,
+			},
-// 				NextUID:            tt.fields.NextUID,
+			wantErr: false,
-// 				Me:                 tt.fields.Me,
+		},
-// 				TokenEndpoint:      tt.fields.TokenEndpoint,
+	}
-// 				AuthEnabled:        tt.fields.AuthEnabled,
+	for _, tt := range tests {
-// 				ticker:             tt.fields.ticker,
+		t.Run(tt.name, func(t *testing.T) {
-// 				quit:               tt.fields.quit,
+			b := &memoryBackend{
-// 				broker:             tt.fields.broker,
+				hubIncomingBackend: tt.fields.hubIncomingBackend,
-// 				pool:               tt.fields.pool,
+				lock:               tt.fields.lock,
-// 			}
+				Channels:           tt.fields.Channels,
-// 			got, err := b.ChannelsCreate(tt.args.name)
+				Feeds:              tt.fields.Feeds,
-// 			if (err != nil) != tt.wantErr {
+				Settings:           tt.fields.Settings,
-// 				t.Errorf("ChannelsCreate() error = %v, wantErr %v", err, tt.wantErr)
+				NextUID:            tt.fields.NextUID,
-// 				return
+				Me:                 tt.fields.Me,
-// 			}
+				TokenEndpoint:      tt.fields.TokenEndpoint,
-// 			if !reflect.DeepEqual(got, tt.want) {
+				AuthEnabled:        tt.fields.AuthEnabled,
-// 				t.Errorf("ChannelsCreate() got = %v, want %v", got, tt.want)
+				ticker:             tt.fields.ticker,
-// 			}
+				quit:               tt.fields.quit,
-// 		})
+				broker:             tt.fields.broker,
-// 	}
+				pool:               tt.fields.pool,
-// }
+			}
-//
+			got, err := b.ChannelsCreate(tt.args.name)
-// func Test_memoryBackend_removeFeed(t *testing.T) {
+			if (err != nil) != tt.wantErr {
-// 	type fields struct {
+				t.Errorf("ChannelsCreate() error = %v, wantErr %v", err, tt.wantErr)
-// 		Channels map[string]microsub.Channel
+				return
-// 		Feeds    map[string][]microsub.Feed
+			}
-// 	}
+			if !reflect.DeepEqual(got, tt.want) {
-// 	type args struct {
+				t.Errorf("ChannelsCreate() got = %v, want %v", got, tt.want)
-// 		feedID string
+			}
-// 	}
+		})
-// 	tests := []struct {
+	}
-// 		name    string
+}
-// 		fields  fields
-// 		args    args
-// 		lens    map[string]int
-// 		wantErr bool
-// 	}{
-// 		{
-// 			name: "remove from channel 1",
-// 			fields: fields{
-// 				Channels: map[string]microsub.Channel{
-// 					"123": {UID: "channel1", Name: "Channel 1"},
-// 					"124": {UID: "channel2", Name: "Channel 2"},
-// 				},
-// 				Feeds: map[string][]microsub.Feed{
-// 					"123": {{Type: "feed", URL: "feed1", Name: "Feed1"}},
-// 					"124": {{Type: "feed", URL: "feed2", Name: "Feed2"}},
-// 				},
-// 			},
-// 			args:    args{feedID: "feed1"},
-// 			lens:    map[string]int{"123": 0, "124": 1},
-// 			wantErr: false,
-// 		},
-// 		{
-// 			name: "remove from channel 2",
-// 			fields: fields{
-// 				Channels: map[string]microsub.Channel{
-// 					"123": {UID: "channel1", Name: "Channel 1"},
-// 					"124": {UID: "channel2", Name: "Channel 2"},
-// 				},
-// 				Feeds: map[string][]microsub.Feed{
-// 					"123": {{Type: "feed", URL: "feed1", Name: "Feed1"}},
-// 					"124": {{Type: "feed", URL: "feed2", Name: "Feed2"}},
-// 				},
-// 			},
-// 			args:    args{feedID: "feed2"},
-// 			lens:    map[string]int{"123": 1, "124": 0},
-// 			wantErr: false,
-// 		},
-// 		{
-// 			name: "remove unknown",
-// 			fields: fields{
-// 				Channels: map[string]microsub.Channel{
-// 					"123": {UID: "channel1", Name: "Channel 1"},
-// 					"124": {UID: "channel2", Name: "Channel 2"},
-// 				},
-// 				Feeds: map[string][]microsub.Feed{
-// 					"123": {{Type: "feed", URL: "feed1", Name: "Feed1"}},
-// 					"124": {{Type: "feed", URL: "feed2", Name: "Feed2"}},
-// 				},
-// 			},
-// 			args:    args{feedID: "feed3"},
-// 			lens:    map[string]int{"123": 1, "124": 1},
-// 			wantErr: false,
-// 		},
-// 		{
-// 			name: "remove from 0 channels",
-// 			fields: fields{
-// 				Channels: map[string]microsub.Channel{},
-// 				Feeds:    map[string][]microsub.Feed{},
-// 			},
-// 			args:    args{feedID: "feed3"},
-// 			lens:    map[string]int{},
-// 			wantErr: false,
-// 		},
-// 		{
-// 			name: "remove from multiple channels",
-// 			fields: fields{
-// 				Channels: map[string]microsub.Channel{
-// 					"123": {UID: "channel1", Name: "Channel 1"},
-// 					"124": {UID: "channel2", Name: "Channel 2"},
-// 				},
-// 				Feeds: map[string][]microsub.Feed{
-// 					"123": {{Type: "feed", URL: "feed1", Name: "Feed1"}},
-// 					"124": {{Type: "feed", URL: "feed1", Name: "Feed1"}},
-// 				},
-// 			},
-// 			args:    args{feedID: "feed1"},
-// 			lens:    map[string]int{"123": 0, "124": 0},
-// 			wantErr: false,
-// 		},
-// 	}
-// 	for _, tt := range tests {
-// 		t.Run(tt.name, func(t *testing.T) {
-// 			b := &memoryBackend{
-// 				Channels: tt.fields.Channels,
-// 				Feeds:    tt.fields.Feeds,
-// 			}
-// 			if err := b.removeFeed(tt.args.feedID); (err != nil) != tt.wantErr {
-// 				t.Errorf("removeFeed() error = %v, wantErr %v", err, tt.wantErr)
-// 			}
-// 			assert.Len(t, b.Channels, len(tt.lens))
-// 			for k, v := range tt.lens {
-// 				assert.Len(t, b.Feeds[k], v)
-// 			}
-// 		})
-// 	}
-// }

cmd/eksterd/micropub.go

@@ -1,26 +1,7 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package main
 
 import (
 	"crypto/sha1"
-	"database/sql"
 	"encoding/json"
 	"fmt"
 	"log"
@@ -59,57 +40,44 @@ func (h *micropubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	err := r.ParseForm()
 	if err != nil {
-		http.Error(w, "bad request", http.StatusBadRequest)
+		http.Error(w, "bad request", 400)
 		return
 	}
 
 	if r.Method == http.MethodPost {
 		var channel string
 
-		sourceID, channel, err := getChannelFromAuthorization(r, conn, h.Backend.database)
+		channel, err = getChannelFromAuthorization(r, conn)
 		if err != nil {
 			log.Println(err)
-			http.Error(w, "unauthorized", http.StatusUnauthorized)
+			http.Error(w, "unauthorized", 401)
 			return
 		}
 
 		// no channel is found
 		if channel == "" {
-			http.Error(w, "bad request, unknown channel", http.StatusBadRequest)
+			http.Error(w, "bad request, unknown channel", 400)
 			return
 		}
 
-		// TODO: We could try to fill the Source of the Item with something, but what?
 		item, err := parseIncomingItem(r)
 		if err != nil {
-			log.Println(err)
+			http.Error(w, err.Error(), 400)
-			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
-		log.Printf("Item published: %s", item.Published)
-		if item.Published == "" {
-			item.Published = time.Now().Format(time.RFC3339)
-		}
 
 		item.Read = false
 		newID, err := generateItemID(conn, channel)
 		if err != nil {
-			log.Println(err)
+			http.Error(w, err.Error(), 500)
-			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
 		}
 		item.ID = newID
 
-		item.Source = &microsub.Source{
+		err = h.Backend.channelAddItemWithMatcher(channel, *item)
-			ID:   fmt.Sprintf("micropub:%d", sourceID),
-			Name: fmt.Sprintf("Source %d", sourceID),
-		}
-
-		_, err = h.Backend.channelAddItemWithMatcher(channel, *item)
 		if err != nil {
 			log.Printf("could not add item to channel %s: %v", channel, err)
 		}
-
 		err = h.Backend.updateChannelUnreadCount(channel)
 		if err != nil {
 			log.Printf("could not update channel unread content %s: %v", channel, err)
@@ -118,14 +86,12 @@ func (h *micropubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "application/json")
 
 		if err = json.NewEncoder(w).Encode(map[string]string{"ok": "1"}); err != nil {
-			log.Println(err)
+			http.Error(w, "internal server error", 500)
-			http.Error(w, "internal server error", http.StatusInternalServerError)
 		}
-
 		return
 	}
 
-	http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+	http.Error(w, "Method not allowed", 405)
 }
 
 func generateItemID(conn redis.Conn, channel string) (string, error) {
@@ -137,57 +103,52 @@ func generateItemID(conn redis.Conn, channel string) (string, error) {
 }
 
 func parseIncomingItem(r *http.Request) (*microsub.Item, error) {
+	var item microsub.Item
+
 	contentType := r.Header.Get("content-type")
 
 	if contentType == "application/jf2+json" {
-		var item microsub.Item
+		dec := json.NewDecoder(r.Body)
-		if err := json.NewDecoder(r.Body).Decode(&item); err != nil {
+		err := dec.Decode(&item)
-			return nil, fmt.Errorf("could not decode request body as %q: %v", contentType, err)
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not decode request body as jf2: %v", err)
 		}
-		return &item, nil
 	} else if contentType == "application/json" {
 		var mfItem microformats.Microformat
-		if err := json.NewDecoder(r.Body).Decode(&mfItem); err != nil {
+		dec := json.NewDecoder(r.Body)
-			return nil, fmt.Errorf("could not decode request body as %q: %v", contentType, err)
+		err := dec.Decode(&mfItem)
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not decode request body as json: %v", err)
 		}
 		author := microsub.Card{}
-		item, ok := jf2.SimplifyMicroformatItem(&mfItem, author)
+		var ok bool
+		item, ok = jf2.SimplifyMicroformatItem(&mfItem, author)
 		if !ok {
 			return nil, fmt.Errorf("could not simplify microformat item to jf2")
 		}
-		return &item, nil
 	} else if contentType == "application/x-www-form-urlencoded" {
-		// TODO: improve handling of form-urlencoded
-		var item microsub.Item
 		content := r.FormValue("content")
 		name := r.FormValue("name")
 		item.Type = "entry"
 		item.Name = name
 		item.Content = &microsub.Content{Text: content}
 		item.Published = time.Now().Format(time.RFC3339)
-		return &item, nil
+	} else {
+		return nil, fmt.Errorf("content-type %s is not supported", contentType)
 	}
-
+	return &item, nil
-	return nil, fmt.Errorf("content-type %q is not supported", contentType)
 }
 
-func getChannelFromAuthorization(r *http.Request, conn redis.Conn, database *sql.DB) (int, string, error) {
+func getChannelFromAuthorization(r *http.Request, conn redis.Conn) (string, error) {
 	// backward compatible
 	sourceID := r.URL.Query().Get("source_id")
 	if sourceID != "" {
-		row := database.QueryRow(`
+		channel, err := redis.String(conn.Do("HGET", "sources", sourceID))
-SELECT s.id as source_id, c.uid
+		if err != nil {
-FROM "sources" AS "s"
+			return "", errors.Wrapf(err, "could not get channel for sourceID: %s", sourceID)
-INNER JOIN "channels" AS "c" ON s.channel_id = c.id
-WHERE "auth_code" = $1
-`, sourceID)
-
-		var channel string
-		var sourceID int
-		if err := row.Scan(&sourceID, &channel); err == sql.ErrNoRows {
-			return 0, "", errors.New("channel not found")
 		}
-		return sourceID, channel, nil
+
+		return channel, nil
 	}
 
 	// full micropub with indieauth
@@ -196,11 +157,11 @@ WHERE "auth_code" = $1
 		token := authHeader[7:]
 		channel, err := redis.String(conn.Do("HGET", "token:"+token, "channel"))
 		if err != nil {
-			return 0, "", errors.Wrap(err, "could not get channel for token")
+			return "", errors.Wrap(err, "could not get channel for token")
 		}
 
-		return 0, channel, nil
+		return channel, nil
 	}
 
-	return 0, "", fmt.Errorf("could not get channel from authorization")
+	return "", fmt.Errorf("could not get channel from authorization")
 }

cmd/eksterd/search.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package main
 
 import (

cmd/eksterd/templates/auth.html

@@ -1,5 +1,6 @@
+{{- /*gotype: p83.nl/go/ekster/cmd/eksterd.authPage*/ -}}
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 <meta charset="utf-8">
 <meta name="viewport" content="width=device-width, initial-scale=1">

cmd/eksterd/templates/channel.html

@@ -1,3 +1,4 @@
+{{- /*gotype: p83.nl/go/ekster/cmd/eksterd.settingsPage*/ -}}
 <!DOCTYPE html>
 <html>
 <head>

cmd/eksterd/templates/index.html

@@ -12,6 +12,8 @@
 <body>
     <section class="section">
         <div class="container">
+
+
             <nav class="navbar" role="navigation" aria-label="main navigation">
                 <div class="navbar-brand">
                     <a class="navbar-item" href="/">
@@ -63,11 +65,6 @@
                     </div>
                 </form>
             {{ end }}
-
-            <p>Copyright (C) 2018 The Ekster authors.<br>
-            This program comes with ABSOLUTELY NO WARRANTY. This is free software,
-            and you are welcome to redistribute it under certain conditions.
-            See <a href="https://www.gnu.org/licenses/gpl-3.0.en.html">Licence</a> for more details.</p>
         </div>
     </section>
 </body>

cmd/jf2test/main.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package main
 
 import (
@@ -50,7 +32,7 @@ func main() {
 	}
 	defer resp.Body.Close()
 
-	items, err := fetch.FeedItems(fetch.FetcherFunc(Fetch), url, resp.Header.Get("Content-Type"), resp.Body)
+	items, err := fetch.FeedItems(Fetch, url, resp.Header.Get("Content-Type"), resp.Body)
 	if err != nil {
 		log.Fatal(err)
 	}

docker-compose.yml

@@ -4,7 +4,7 @@ services:
     image: "redis:5"
 
   database:
-    image: postgres:14
+    image: postgres
     volumes:
     - database-data:/var/lib/postgresql/data
     environment:
@@ -14,16 +14,21 @@ services:
       POSTGRES_HOST_AUTH_METHOD: trust
 
   web:
-    image: ubuntu
+    image: "pstuifzand/ekster:alpine"
-    working_dir: /app
+    working_dir: /opt/microsub
+    links:
+      - redis:redis
     volumes:
+      - microsub-data:/opt/microsub
+      - ./templates:/app/templates
       - ./eksterd:/app/eksterd
-      - ./backend.json:/app/backend.json
     entrypoint: /app/eksterd
-    command: -auth=false -port 80
+    command: -auth=false -port 80 -templates templates
     ports:
       - 8089:80
     environment:
+      - "FEEDBIN_USER="
+      - "FEEDBIN_PASS="
       - "EKSTER_BASEURL=http://localhost:8089/"
       - "EKSTER_TEMPLATES=/app/templates"
 

go.mod

@@ -3,16 +3,15 @@ module p83.nl/go/ekster
 go 1.16
 
 require (
+	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751
 	github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394
-	github.com/blevesearch/bleve/v2 v2.0.3
+	github.com/blevesearch/bleve/v2 v2.0.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/gilliek/go-opml v1.0.0
-	github.com/golang-migrate/migrate/v4 v4.15.1
 	github.com/gomodule/redigo v1.8.2
 	github.com/lib/pq v1.10.1
 	github.com/pkg/errors v0.9.1
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/testify v1.5.1
-	golang.org/x/net v0.0.0-20211013171255-e13a2654a71e
+	golang.org/x/net v0.0.0-20200707034311-ab3426394381
-	golang.org/x/text v0.3.7
 	willnorris.com/go/microformats v1.1.0
 )

pkg/auth/types.go

@@ -1,24 +1,6 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package auth
 
-// Auther checks of the token in the head is accepted and fills TokenResponse
+// Auther
 type Auther interface {
 	AuthTokenAccepted(header string, r *TokenResponse) bool
 }

pkg/client/requests.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package client
 
 import (
@@ -106,7 +88,7 @@ func (c *Client) microsubPostRequest(action string, args map[string]string) (*ht
 
 	if res.StatusCode != 200 {
 		msg, _ := ioutil.ReadAll(res.Body)
-		return nil, fmt.Errorf("unsuccessful response: %d: %q", res.StatusCode, strings.TrimSpace(string(msg)))
+		return nil, fmt.Errorf("unsuccessful response: %d: %q", res.StatusCode, string(msg))
 	}
 
 	return res, err
@@ -134,7 +116,7 @@ func (c *Client) microsubPostFormRequest(action string, args map[string]string,
 
 	if res.StatusCode != 200 {
 		msg, _ := ioutil.ReadAll(res.Body)
-		return nil, fmt.Errorf("unsuccessful response: %d: %q", res.StatusCode, strings.TrimSpace(string(msg)))
+		return nil, fmt.Errorf("unsuccessful response: %d: %q", res.StatusCode, string(msg))
 	}
 
 	return res, err
@@ -198,7 +180,7 @@ func (c *Client) TimelineGet(before, after, channel string) (microsub.Timeline,
 func (c *Client) PreviewURL(url string) (microsub.Timeline, error) {
 	args := make(map[string]string)
 	args["url"] = url
-	res, err := c.microsubPostRequest("preview", args)
+	res, err := c.microsubGetRequest("preview", args)
 	if err != nil {
 		return microsub.Timeline{}, err
 	}

pkg/fetch/fetch.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 // Package fetch provides an API for fetching information about urls.
 package fetch
 
@@ -43,7 +25,7 @@ import (
 )
 
 // FeedHeader returns a new microsub.Feed with the information parsed from body.
-func FeedHeader(fetcher Fetcher, fetchURL, contentType string, body io.Reader) (microsub.Feed, error) {
+func FeedHeader(fetcher FetcherFunc, fetchURL, contentType string, body io.Reader) (microsub.Feed, error) {
 	log.Printf("ProcessContent %s\n", fetchURL)
 	log.Println("Found " + contentType)
 
@@ -56,7 +38,7 @@ func FeedHeader(fetcher Fetcher, fetchURL, contentType string, body io.Reader) (
 		author, ok := jf2.SimplifyMicroformatDataAuthor(data)
 		if !ok {
 			if strings.HasPrefix(author.URL, "http") {
-				resp, err := fetcher.Fetch(author.URL)
+				resp, err := fetcher(author.URL)
 				if err != nil {
 					return feed, err
 				}
@@ -66,9 +48,6 @@ func FeedHeader(fetcher Fetcher, fetchURL, contentType string, body io.Reader) (
 				md := microformats.Parse(resp.Body, u)
 
 				author, ok = jf2.SimplifyMicroformatDataAuthor(md)
-				if !ok {
-					log.Println("Could not simplify the author")
-				}
 			}
 		}
 
@@ -129,7 +108,7 @@ func FeedHeader(fetcher Fetcher, fetchURL, contentType string, body io.Reader) (
 }
 
 // FeedItems returns the items from the url, parsed from body.
-func FeedItems(fetcher Fetcher, fetchURL, contentType string, body io.Reader) ([]microsub.Item, error) {
+func FeedItems(fetcher FetcherFunc, fetchURL, contentType string, body io.Reader) ([]microsub.Item, error) {
 	log.Printf("ProcessContent %s\n", fetchURL)
 	log.Println("Found " + contentType)
 

pkg/fetch/fetch_test.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package fetch
 
 import (
@@ -41,7 +23,7 @@ func TestFeedHeader(t *testing.T) {
 </body>
 </html>
 `
-	feed, err := FeedHeader(FetcherFunc(fetcher), "https://example.com/", "text/html", strings.NewReader(doc))
+	feed, err := FeedHeader(fetcher, "https://example.com/", "text/html", strings.NewReader(doc))
 	if assert.NoError(t, err) {
 		assert.Equal(t, "feed", feed.Type)
 		assert.Equal(t, "Title", feed.Name)

pkg/fetch/fetcher.go

@@ -1,34 +1,6 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package fetch
 
 import "net/http"
 
-// Fetcher fetches urls
-type Fetcher interface {
-	Fetch(url string) (*http.Response, error)
-}
-
 // FetcherFunc is a function that fetches an url
 type FetcherFunc func(url string) (*http.Response, error)
-
-// Fetch fetches an url and returns a response or error
-func (ff FetcherFunc) Fetch(url string) (*http.Response, error) {
-	return ff(url)
-}

pkg/indieauth/auth.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package indieauth
 
 import (

pkg/jf2/reflect.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package jf2
 
 import (

pkg/jf2/simplify.go

@@ -1,22 +1,21 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 // Package jf2 converts microformats to JF2
+/*
+   ekster - microsub server
+   Copyright (C) 2018  Peter Stuifzand
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
 package jf2
 
 import (
@@ -89,14 +88,6 @@ func simplifyContent(k string, v []interface{}) *microsub.Content {
 // CleanHTML removes white-space:pre from html
 func CleanHTML(s string) (string, error) {
 	doc, err := html.Parse(strings.NewReader(s))
-	if err != nil {
-		return "", err
-	}
-
-	whitespaceRegex, err := regexp.Compile(`white-space:\s*pre`)
-	if err != nil {
-		return "", err
-	}
 
 	if err != nil {
 		return "", err
@@ -110,7 +101,7 @@ func CleanHTML(s string) (string, error) {
 				if a.Key != "style" {
 					continue
 				}
-				if whitespaceRegex.MatchString(a.Val) {
+				if m, err := regexp.MatchString("white-space:\\s*pre", a.Val); err == nil && m {
 					removeIndex = i
 					break
 				}

pkg/jf2/simplify_test.go

@@ -1,20 +1,20 @@
 /*
- *  Ekster is a microsub server
+   ekster - microsub server
- *  Copyright (c) 2021 The Ekster authors
+   Copyright (C) 2018  Peter Stuifzand
- *
+
- * This program is free software: you can redistribute it and/or modify
+   This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
+   it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
+   the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
+   (at your option) any later version.
- *
+
- * This program is distributed in the hope that it will be useful,
+   This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
+   GNU General Public License for more details.
- *
+
- * You should have received a copy of the GNU General Public License
+   You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
+*/
 package jf2_test
 
 import (

pkg/jsonfeed/jsonfeed.go

@@ -1,22 +1,18 @@
-/*
+// Package jsonfeed contains the types and a parse function for JSON feeds.
- *  Ekster is a microsub server
+// Copyright (C) 2018  Peter Stuifzand
- *  Copyright (c) 2021 The Ekster authors
+//
- *
+// This program is free software: you can redistribute it and/or modify
- * This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
- * it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
- * the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
- * (at your option) any later version.
+//
- *
+// This program is distributed in the hope that it will be useful,
- * This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
- * GNU General Public License for more details.
+//
- *
+// You should have received a copy of the GNU General Public License
- * You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-// Package jsonfeed parses feeds in the jsonfeed format
 package jsonfeed
 
 import (

pkg/jsonfeed/jsonfeed_test.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package jsonfeed
 
 import (

pkg/microsub/database.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package microsub
 
 import (

pkg/microsub/protocol.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 // Package microsub describes the protocol methods of the Microsub protocol
 package microsub
 
@@ -100,15 +82,6 @@ type Item struct {
 	Refs       map[string]Item `json:"refs,omitempty"`
 	ID         string          `json:"_id,omitempty"`
 	Read       bool            `json:"_is_read"`
-	Source     *Source         `json:"_source,omitempty"`
-}
-
-// Source is an Item source
-type Source struct {
-	ID    string `json:"_id"`
-	URL   string `json:"url"`
-	Name  string `json:"name"`
-	Photo string `json:"photo"`
 }
 
 // Pagination contains information about paging

pkg/microsub/protocol_test.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package microsub
 
 import (

pkg/rss/ISO-8859-1.go

@@ -37,7 +37,7 @@ func (cs *charsetISO88591er) ReadByte() (b byte, err error) {
 
 func (cs *charsetISO88591er) Read(p []byte) (int, error) {
 	// Use ReadByte method.
-	return 0, errors.New("use ReadByte()")
+	return 0, errors.New("Use ReadByte()")
 }
 
 func isCharsetISO88591(charset string) bool {

pkg/rss/rss.go

@@ -303,7 +303,7 @@ type Enclosure struct {
 // Get uses http.Get to fetch an enclosure.
 func (e *Enclosure) Get() (io.ReadCloser, error) {
 	if e == nil || e.URL == "" {
-		return nil, errors.New("no enclosure")
+		return nil, errors.New("No enclosure")
 	}
 
 	res, err := http.Get(e.URL)
@@ -325,7 +325,7 @@ type Image struct {
 // Get uses http.Get to fetch an image.
 func (i *Image) Get() (io.ReadCloser, error) {
 	if i == nil || i.URL == "" {
-		return nil, errors.New("no image")
+		return nil, errors.New("No image")
 	}
 
 	res, err := http.Get(i.URL)

pkg/rss/rss_1.0.go

@@ -5,10 +5,8 @@ import (
 	"encoding/xml"
 	"fmt"
 	"sort"
+	"strings"
 	"time"
-
-	"golang.org/x/text/cases"
-	"golang.org/x/text/language"
 )
 
 func parseRSS1(data []byte) (*Feed, error) {
@@ -31,8 +29,6 @@ func parseRSS1(data []byte) (*Feed, error) {
 	out.Description = channel.Description
 	out.Link = channel.Link
 	out.Image = channel.Image.Image()
-
-	titleCaser := cases.Title(language.English)
 	if channel.MinsToLive != 0 {
 		sort.Ints(channel.SkipHours)
 		next := time.Now().Add(time.Duration(channel.MinsToLive) * time.Minute)
@@ -45,7 +41,7 @@ func parseRSS1(data []byte) (*Feed, error) {
 		for trying {
 			trying = false
 			for _, day := range channel.SkipDays {
-				if titleCaser.String(day) == next.Weekday().String() {
+				if strings.Title(day) == next.Weekday().String() {
 					next.Add(time.Duration(24-next.Hour()) * time.Hour)
 					trying = true
 					break

pkg/rss/rss_2.0.go

@@ -5,10 +5,8 @@ import (
 	"encoding/xml"
 	"fmt"
 	"sort"
+	"strings"
 	"time"
-
-	"golang.org/x/text/cases"
-	"golang.org/x/text/language"
 )
 
 func parseRSS2(data []byte) (*Feed, error) {
@@ -40,7 +38,6 @@ func parseRSS2(data []byte) (*Feed, error) {
 
 	out.Image = channel.Image.Image()
 	if channel.MinsToLive != 0 {
-		titleCaser := cases.Title(language.English)
 		sort.Ints(channel.SkipHours)
 		next := time.Now().Add(time.Duration(channel.MinsToLive) * time.Minute)
 		for _, hour := range channel.SkipHours {
@@ -52,7 +49,7 @@ func parseRSS2(data []byte) (*Feed, error) {
 		for trying {
 			trying = false
 			for _, day := range channel.SkipDays {
-				if titleCaser.String(day) == next.Weekday().String() {
+				if strings.Title(day) == next.Weekday().String() {
 					next.Add(time.Duration(24-next.Hour()) * time.Hour)
 					trying = true
 					break

pkg/rss/rss_2.0_test.go

@@ -1,6 +1,7 @@
 package rss
 
 import (
+	"fmt"
 	"io/ioutil"
 	"path/filepath"
 	"testing"
@@ -76,7 +77,7 @@ func TestParseItemDateOK(t *testing.T) {
 			t.Fatalf("Parsing %s: %v", name, err)
 		}
 
-		if feed.Items[0].Date.String() != want {
+		if fmt.Sprintf("%s", feed.Items[0].Date) != want {
 			t.Errorf("%s: got %q, want %q", name, feed.Items[0].Date, want)
 		}
 	}
@@ -99,7 +100,7 @@ func TestParseItemDateFailure(t *testing.T) {
 			t.Fatalf("Parsing %s: %v", name, err)
 		}
 
-		if feed.Items[1].Date.String() != want {
+		if fmt.Sprintf("%s", feed.Items[1].Date) != want {
 			t.Errorf("%s: got %q, want %q", name, feed.Items[1].Date, want)
 		}
 

pkg/server/microsub.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 /*
 Package server contains the microsub server itself. It implements http.Handler.
 It follows the spec at https://indieweb.org/Microsub-spec.
@@ -34,7 +16,7 @@ import (
 )
 
 var (
-	entryRegex = regexp.MustCompile(`^entry\[\d+\]$`)
+	entryRegex = regexp.MustCompile("^entry\\[\\d+\\]$")
 )
 
 // Constants used for the responses
@@ -54,7 +36,7 @@ func respondJSON(w http.ResponseWriter, value interface{}) {
 	w.Header().Add("Content-Type", OutputContentType)
 	err := jw.Encode(value)
 	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
+		http.Error(w, err.Error(), 500)
 	}
 }
 
@@ -87,8 +69,7 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		if action == "channels" {
 			channels, err := h.backend.ChannelsGetList()
 			if err != nil {
-				log.Println(err)
+				http.Error(w, err.Error(), 500)
-				http.Error(w, err.Error(), http.StatusInternalServerError)
 				return
 			}
 			respondJSON(w, map[string][]microsub.Channel{
@@ -97,16 +78,14 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		} else if action == "timeline" {
 			timeline, err := h.backend.TimelineGet(values.Get("before"), values.Get("after"), values.Get("channel"))
 			if err != nil {
-				log.Println(err)
+				http.Error(w, err.Error(), 500)
-				http.Error(w, err.Error(), http.StatusInternalServerError)
 				return
 			}
 			respondJSON(w, timeline)
 		} else if action == "preview" {
 			timeline, err := h.backend.PreviewURL(values.Get("url"))
 			if err != nil {
-				log.Println(err)
+				http.Error(w, err.Error(), 500)
-				http.Error(w, err.Error(), http.StatusInternalServerError)
 				return
 			}
 			respondJSON(w, timeline)
@@ -114,8 +93,7 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			channel := values.Get("channel")
 			following, err := h.backend.FollowGetList(channel)
 			if err != nil {
-				log.Println(err)
+				http.Error(w, err.Error(), 500)
-				http.Error(w, err.Error(), http.StatusInternalServerError)
 				return
 			}
 			respondJSON(w, map[string][]microsub.Feed{
@@ -124,8 +102,7 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		} else if action == "events" {
 			events, err := h.backend.Events()
 			if err != nil {
-				log.Println(err)
+				http.Error(w, "could not start sse connection", 500)
-				http.Error(w, "could not start sse connection", http.StatusInternalServerError)
 			}
 
 			// Remove this client from the map of connected clients
@@ -135,7 +112,7 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			}()
 
 			// Listen to connection close and un-register messageChan
-			notify := r.Context().Done()
+			notify := w.(http.CloseNotifier).CloseNotify()
 
 			go func() {
 				<-notify
@@ -145,10 +122,10 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			err = sse.WriteMessages(w, events)
 			if err != nil {
 				log.Println(err)
-				http.Error(w, "internal server error", http.StatusInternalServerError)
+				http.Error(w, "internal server error", 500)
 			}
 		} else {
-			http.Error(w, fmt.Sprintf("unknown action %s", action), http.StatusBadRequest)
+			http.Error(w, fmt.Sprintf("unknown action %s\n", action), 400)
 			return
 		}
 		return
@@ -164,8 +141,7 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			if method == "delete" {
 				err := h.backend.ChannelsDelete(uid)
 				if err != nil {
-					log.Println(err)
+					http.Error(w, err.Error(), 500)
-					http.Error(w, err.Error(), http.StatusInternalServerError)
 					return
 				}
 				respondJSON(w, []string{})
@@ -175,16 +151,14 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			if uid == "" {
 				channel, err := h.backend.ChannelsCreate(name)
 				if err != nil {
-					log.Println(err)
+					http.Error(w, err.Error(), 500)
-					http.Error(w, err.Error(), http.StatusInternalServerError)
 					return
 				}
 				respondJSON(w, channel)
 			} else if name != "" {
 				channel, err := h.backend.ChannelsUpdate(uid, name)
 				if err != nil {
-					log.Println(err)
+					http.Error(w, err.Error(), 500)
-					http.Error(w, err.Error(), http.StatusInternalServerError)
 					return
 				}
 				respondJSON(w, channel)
@@ -195,7 +169,7 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			// h.HubIncomingBackend.CreateFeed(url, uid)
 			feed, err := h.backend.FollowURL(uid, url)
 			if err != nil {
-				http.Error(w, err.Error(), http.StatusInternalServerError)
+				http.Error(w, err.Error(), 500)
 				return
 			}
 			respondJSON(w, feed)
@@ -204,17 +178,10 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			url := values.Get("url")
 			err := h.backend.UnfollowURL(uid, url)
 			if err != nil {
-				http.Error(w, err.Error(), http.StatusInternalServerError)
+				http.Error(w, err.Error(), 500)
 				return
 			}
 			respondJSON(w, []string{})
-		} else if action == "preview" {
-			timeline, err := h.backend.PreviewURL(values.Get("url"))
-			if err != nil {
-				http.Error(w, err.Error(), http.StatusInternalServerError)
-				return
-			}
-			respondJSON(w, timeline)
 		} else if action == "search" {
 			query := values.Get("query")
 			channel := values.Get("channel")
@@ -239,7 +206,6 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 					})
 					return
 				}
-				log.Printf("Searching for %s in %s (%d results)", query, channel, len(items))
 				respondJSON(w, map[string]interface{}{
 					"query": query,
 					"items": items,
@@ -269,21 +235,22 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 				if len(markAsRead) > 0 {
 					err := h.backend.MarkRead(channel, markAsRead)
 					if err != nil {
-						http.Error(w, err.Error(), http.StatusInternalServerError)
+						http.Error(w, err.Error(), 500)
 						return
 					}
 				} else {
 					log.Println("No uids specified for mark read")
 				}
 			} else {
-				http.Error(w, fmt.Sprintf("unknown method in timeline %s\n", method), http.StatusInternalServerError)
+				http.Error(w, fmt.Sprintf("unknown method in timeline %s\n", method), 500)
 				return
 			}
 
 			respondJSON(w, []string{})
 		} else {
-			http.Error(w, fmt.Sprintf("unknown action %s\n", action), http.StatusBadRequest)
+			http.Error(w, fmt.Sprintf("unknown action %s\n", action), 400)
 		}
 		return
 	}
+	return
 }

pkg/server/microsub_test.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package server
 
 import (

pkg/server/null.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package server
 
 import (

pkg/sse/events.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package sse
 
 import (

pkg/timeline/null.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package timeline
 
 import "p83.nl/go/ekster/pkg/microsub"

pkg/timeline/postgres.go

@@ -1,31 +1,10 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package timeline
 
 import (
 	"context"
-	"crypto/sha256"
 	"database/sql"
-	"encoding/hex"
 	"fmt"
 	"log"
-	"strconv"
 	"strings"
 	"time"
 
@@ -54,10 +33,46 @@ func (p *postgresStream) Init() error {
 		return fmt.Errorf("database ping failed: %w", err)
 	}
 
-	row := conn.QueryRowContext(ctx, `SELECT "id" FROM "channels" WHERE "uid" = $1`, p.channel)
+	_, err = conn.ExecContext(ctx, `
+CREATE TABLE IF NOT EXISTS "channels" (
+    "id" int primary key generated always as identity,
+    "name" varchar(255) unique,
+    "created_at" timestamp DEFAULT current_timestamp
+);
+`)
+	if err != nil {
+		return fmt.Errorf("create channels table failed: %w", err)
+	}
+
+	_, err = conn.ExecContext(ctx, `
+CREATE TABLE IF NOT EXISTS "items" (
+    "id" int primary key generated always as identity,
+    "channel_id" int references "channels" on delete cascade,
+    "uid" varchar(512) not null unique,
+    "is_read" int default 0,
+    "data" json,
+    "created_at" timestamp DEFAULT current_timestamp,
+    "updated_at" timestamp,
+    "published_at" timestamp
+);
+`)
+	if err != nil {
+		return fmt.Errorf("create items table failed: %w", err)
+	}
+
+	_, err = conn.ExecContext(ctx, `INSERT INTO "channels" ("name", "created_at") VALUES ($1, DEFAULT)
+ 		ON CONFLICT DO NOTHING`, p.channel)
+	if err != nil {
+		return fmt.Errorf("create channel failed: %w", err)
+	}
+
+	row := conn.QueryRowContext(ctx, `SELECT "id" FROM "channels" WHERE "name" = $1`, p.channel)
+	if row == nil {
+		return fmt.Errorf("fetch channel failed: %w", err)
+	}
 	err = row.Scan(&p.channelID)
-	if err == sql.ErrNoRows {
+	if err != nil {
-		return fmt.Errorf("channel %s not found: %w", p.channel, err)
+		return fmt.Errorf("fetch channel failed while scanning: %w", err)
 	}
 
 	return nil
@@ -86,16 +101,16 @@ WHERE "channel_id" = $1
 			log.Println(err)
 		} else {
 			args = append(args, b)
-			qb.WriteString(` AND "published_at" > $2`)
+			qb.WriteString(` AND "published_at" < $2`)
 		}
 	} else if after != "" {
 		b, err := time.Parse(time.RFC3339, after)
 		if err == nil {
 			args = append(args, b)
-			qb.WriteString(` AND "published_at" < $2`)
+			qb.WriteString(` AND "published_at" > $2`)
 		}
 	}
-	qb.WriteString(` ORDER BY "published_at" DESC LIMIT 20`)
+	qb.WriteString(` ORDER BY "published_at" DESC LIMIT 10`)
 
 	rows, err := conn.QueryContext(context.Background(), qb.String(), args...)
 	if err != nil {
@@ -139,12 +154,9 @@ WHERE "channel_id" = $1
 		return tl, err
 	}
 
-	if len(tl.Items) > 0 && hasMoreBefore(conn, tl.Items[0].Published) {
+	// TODO: should only be set of there are more items available
-		tl.Paging.Before = tl.Items[0].Published
+	tl.Paging.Before = last
-	}
+	// tl.Paging.After = last
-	if hasMoreAfter(conn, last) {
-		tl.Paging.After = last
-	}
 
 	if tl.Items == nil {
 		tl.Items = []microsub.Item{}
@@ -153,24 +165,6 @@ WHERE "channel_id" = $1
 	return tl, nil
 }
 
-func hasMoreBefore(conn *sql.Conn, before string) bool {
-	row := conn.QueryRowContext(context.Background(), `SELECT COUNT(*) FROM "items" WHERE "published_at" > $1`, before)
-	var count int
-	if err := row.Scan(&count); err == sql.ErrNoRows {
-		return false
-	}
-	return count > 0
-}
-
-func hasMoreAfter(conn *sql.Conn, after string) bool {
-	row := conn.QueryRowContext(context.Background(), `SELECT COUNT(*) FROM "items" WHERE "published_at" < $1`, after)
-	var count int
-	if err := row.Scan(&count); err == sql.ErrNoRows {
-		return false
-	}
-	return count > 0
-}
-
 // Count
 func (p *postgresStream) Count() (int, error) {
 	ctx := context.Background()
@@ -179,12 +173,16 @@ func (p *postgresStream) Count() (int, error) {
 		return -1, err
 	}
 	defer conn.Close()
-	var count int
 	row := conn.QueryRowContext(context.Background(), `SELECT COUNT(*) FROM items WHERE channel_id = $1 AND "is_read" = 0`, p.channelID)
-	err = row.Scan(&count)
+	if row == nil {
-	if err != nil && err == sql.ErrNoRows {
 		return 0, nil
 	}
+	var count int
+	err = row.Scan(&count)
+	if err != nil {
+		return -1, err
+	}
+
 	return count, nil
 }
 
@@ -205,34 +203,14 @@ func (p *postgresStream) AddItem(item microsub.Item) (bool, error) {
 		}
 		t = t2
 	}
-	if item.ID == "" {
-		// FIXME: This won't work when we receive the item multiple times
-		h := sha256.Sum256([]byte(fmt.Sprintf("%s:%d", p.channel, time.Now().UnixNano())))
-		item.UID = hex.EncodeToString(h[:])
-	}
-
-	var optFeedID sql.NullInt64
-	if item.Source == nil || item.Source.ID == "" {
-		optFeedID.Valid = false
-		optFeedID.Int64 = 0
-	} else {
-		feedID, err := strconv.ParseInt(item.Source.ID, 10, 64)
-		if err != nil {
-			optFeedID.Valid = false
-			optFeedID.Int64 = 0
-		} else {
-			optFeedID.Valid = true
-			optFeedID.Int64 = feedID
-		}
-	}
 
 	result, err := conn.ExecContext(context.Background(), `
-INSERT INTO "items" ("channel_id", "feed_id", "uid", "data", "published_at", "created_at")
+INSERT INTO "items" ("channel_id", "uid", "data", "published_at", "created_at")
-VALUES ($1, $2, $3, $4, $5, DEFAULT)
+VALUES ($1, $2, $3, $4, DEFAULT)
 ON CONFLICT ON CONSTRAINT "items_uid_key" DO NOTHING
-`, p.channelID, optFeedID, item.ID, &item, t)
+`, p.channelID, item.ID, &item, t)
 	if err != nil {
-		return false, fmt.Errorf("insert item: %w", err)
+		return false, fmt.Errorf("while adding item: %w", err)
 	}
 	c, err := result.RowsAffected()
 	if err != nil {

pkg/timeline/redisset.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package timeline
 
 import (
@@ -132,12 +114,6 @@ func (timeline *redisSortedSetTimeline) AddItem(item microsub.Item) (bool, error
 		item.Published = time.Now().Format(time.RFC3339)
 	}
 
-	// Fix date when it almost matches with RFC3339, except the colon in the timezone
-	format := "2006-01-02T15:04:05Z0700"
-	if parsedDate, err := time.Parse(format, item.Published); err == nil {
-		item.Published = parsedDate.Format(time.RFC3339)
-	}
-
 	data, err := json.Marshal(item)
 	if err != nil {
 		return false, fmt.Errorf("couldn't marshal item for redis: %s", err)

pkg/timeline/redisstreams.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package timeline
 
 import (

pkg/timeline/timeline.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 // Package timeline contains different types of timeline backends.
 //
 // "sorted-set" uses Redis sorted sets as a backend

pkg/util/algorithm.go

@@ -1,26 +1,7 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package util
 
 import "reflect"
 
-// Rotate rotates values of an array, between index f and l with midpoint k
 func Rotate(a interface{}, f, k, l int) int {
 	swapper := reflect.Swapper(a)
 	if f == k {
@@ -57,7 +38,6 @@ func Rotate(a interface{}, f, k, l int) int {
 	return ret
 }
 
-// StablePartition partitions elements of the array between indices f and l according to predicate p
 func StablePartition(a interface{}, f, l int, p func(i int) bool) int {
 	n := l - f
 
@@ -68,7 +48,7 @@ func StablePartition(a interface{}, f, l int, p func(i int) bool) int {
 	if n == 1 {
 		t := f
 		if p(f) {
-			t++
+			t += 1
 		}
 		return t
 	}

pkg/util/randkey.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package util
 
 import (
@@ -24,7 +6,6 @@ import (
 
 const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
 
-// RandStringBytes generates a random string of n characters
 func RandStringBytes(n int) string {
 	b := make([]byte, n)
 	for i := range b {

pkg/websub/signature.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package websub
 
 import (

pkg/websub/signature_test.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package websub
 
 import (

pkg/websub/subscribe.go

@@ -1,21 +1,3 @@
-/*
- *  Ekster is a microsub server
- *  Copyright (c) 2021 The Ekster authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 package websub
 
 import (

templates/auth.html

@@ -0,0 +1,100 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Ekster</title>
+<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.1/css/bulma.min.css">
+</head>
+<body>
+    <section class="section">
+        <div class="container">
+
+
+            <nav class="navbar" role="navigation" aria-label="main navigation">
+                <div class="navbar-brand">
+                    <a class="navbar-item" href="/">
+                        Ekster
+                    </a>
+
+                    <a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false" data-target="menu">
+                        <span aria-hidden="true"></span>
+                        <span aria-hidden="true"></span>
+                        <span aria-hidden="true"></span>
+                    </a>
+                </div>
+
+                {{ if .Session.LoggedIn }}
+                    <div id="menu" class="navbar-menu">
+                        <a class="navbar-item" href="/settings">
+                            Settings
+                        </a>
+                        <a class="navbar-item" href="/logs">
+                            Logs
+                        </a>
+                        <a class="navbar-item" href="{{ .Session.Me }}">
+                            Profile
+                        </a>
+                    </div>
+                {{ end }}
+            </nav>
+
+            <h1 class="title">Ekster - Microsub server</h1>
+
+            <div class="box">
+                <form action="/auth/approve" method="post">
+                    <input type="hidden" name="state" value="{{ .State }}" />
+
+                    <div class="field">
+                        <label class="label">Client ID</label>
+                        <div class="control">
+                            <p>{{ .ClientID }}</p>
+                        </div>
+                        <div class="control">
+                            <p>{{ .App.Name }}</p>
+                        </div>
+                        <div class="control">
+                            <p><img src="{{ .App.IconURL }}" /></p>
+                        </div>
+                    </div>
+
+                    <div class="field">
+                        <label class="label">RedirectURI</label>
+                        <div class="control">
+                            <p>{{ .RedirectURI }}</p>
+                        </div>
+                    </div>
+                    
+                    <div class="field">
+                        <label class="label">Scope</label>
+                        <div class="control">
+                            <p>{{ .Scope }}</p>
+                        </div>
+                    </div>
+
+                    <div class="field">
+                        <label class="label">Select a channel</label>
+                        <div class="control">
+                            <div class="select">
+                                <select name="channel">
+                                    {{ range .Channels }}
+                                        <option value="{{ .UID }}">{{ .Name }}</option>
+                                    {{ end }}
+                                </select>
+                            </div>
+                        </div>
+                    </div>
+
+                    <div class="field">
+                        <div class="control">
+                            <button type="submit" name="accept" value="approve" class="button is-primary">
+                                Approve
+                            </button>
+                        </div>
+                    </div>
+                </form>
+            </div>
+        </div>
+    </section>
+</body>
+</html>

templates/base.html

@@ -0,0 +1,49 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>{{template "title" .}}</title>
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.1/css/bulma.min.css">
+</head>
+<body>
+<section class="section">
+    <div class="container">
+        <nav class="navbar" role="navigation" aria-label="main navigation">
+            <div class="navbar-brand">
+                <a class="navbar-item" href="/">
+                    Ekster
+                </a>
+
+                <a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false" data-target="menu">
+                    <span aria-hidden="true"></span>
+                    <span aria-hidden="true"></span>
+                    <span aria-hidden="true"></span>
+                </a>
+            </div>
+
+            {{ if .Session.LoggedIn }}
+                <div id="menu" class="navbar-menu">
+                    <a class="navbar-item" href="/settings">
+                        Settings
+                    </a>
+                    <a class="navbar-item" href="/logs">
+                        Logs
+                    </a>
+                    <a class="navbar-item" href="{{ .Session.Me }}">
+                        Profile
+                    </a>
+                </div>
+            {{ end }}
+        </nav>
+
+        <h1 class="title">Ekster - Microsub server</h1>
+
+        {{template "content" .}}
+    </div>
+</section>
+</body>
+</html>
+{{define "header"}}{{end}}
+{{define "content"}}{{end}}
+{{define "footer"}}{{end}}

templates/channel.html

@@ -0,0 +1,122 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Ekster</title>
+<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.1/css/bulma.min.css">
+</head>
+<body>
+    <section class="section">
+        <div class="container">
+            <nav class="navbar" role="navigation" aria-label="main navigation">
+                <div class="navbar-brand">
+                    <a class="navbar-item" href="/">
+                        Ekster
+                    </a>
+
+                    <a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false" data-target="menu">
+                        <span aria-hidden="true"></span>
+                        <span aria-hidden="true"></span>
+                        <span aria-hidden="true"></span>
+                    </a>
+                </div>
+
+                {{ if .Session.LoggedIn }}
+                    <div id="menu" class="navbar-menu">
+                        <a class="navbar-item" href="/settings">
+                            Settings
+                        </a>
+                        <a class="navbar-item" href="/logs">
+                            Logs
+                        </a>
+                        <a class="navbar-item" href="{{ .Session.Me }}">
+                            Profile
+                        </a>
+                    </div>
+                {{ end }}
+            </nav>
+
+            <h1 class="title">Ekster - Microsub server</h1>
+
+            {{ $channel := .CurrentChannel }}
+
+            <nav class="breadcrumb" aria-label="breadcrumbs">
+                <ul>
+                    <li><a href="/settings">Settings</a></li>
+                    <li class="is-active"><a href="/setttings/channel?uid={{ .CurrentChannel }}">{{ $channel.Name }}</a></li>
+                </ul>
+            </nav>
+
+            <h2 class="subtitle is-2">{{ $channel.Name }}</h2>
+
+            <div class="columns">
+                <div class="column">
+                    <h3 class="title is-4">Settings</h3>
+                    <form action="/settings/channel" method="post">
+                        <input type="hidden" name="uid" value="{{ .CurrentChannel.UID }}" />
+                        <div class="field">
+                            <label class="label" for="exclude_regex">Blocking Regex</label>
+                            <div class="control">
+                                <input type="text" class="input" id="exclude_regex" name="exclude_regex" value="{{ .CurrentSetting.ExcludeRegex }}" placeholder="enter regex to block" />
+                            </div>
+                        </div>
+                        <div class="field">
+                            <label class="label" for="include_regex">Tracking Regex</label>
+                            <div class="control">
+                                <input type="text" class="input" id="include_regex" name="include_regex" value="{{ .CurrentSetting.IncludeRegex }}" placeholder="enter regex to track items" />
+                            </div>
+                        </div>
+                        <div class="field">
+                            <label class="label" for="type">Channel Type</label>
+                            <div class="control">
+                                <div class="select">
+                                    <select name="type" id="type">
+                                        <option value="null" {{if eq (.CurrentSetting.ChannelType) "null" }}selected{{end}}>Null</option>
+                                        <option value="sorted-set" {{if eq (.CurrentSetting.ChannelType) "sorted-set" }}selected{{end}}>Sorted Set</option>
+                                        <option value="stream" {{if eq (.CurrentSetting.ChannelType) "stream" }}selected{{end}}>Streams</option>
+                                        <option value="postgres-stream" {{if eq (.CurrentSetting.ChannelType) "postgres-stream" }}selected{{end}}>Postgres Stream</option>
+                                    </select>
+                                </div>
+                            </div>
+                        </div>
+                        <div class="field">
+                            <label for="exclude_type" class="label">Exclude Types</label>
+                            <div class="control">
+                                <div class="select is-multiple">
+                                    <select name="exclude_type" id="exclude_type" multiple>
+                                        {{ range $key, $excluded := $.ExcludedTypes }}
+                                            <option value="{{ $key }}" {{ if $excluded }}selected="selected"{{ end }}>{{ index $.ExcludedTypeNames $key }}</option>
+                                        {{ end }}
+                                    </select>
+                                </div>
+                            </div>
+                        </div>
+                        <div class="field">
+                            <div class="control">
+                                <button type="submit" class="button is-primary">Save</button>
+                            </div>
+                        </div>
+                    </form>
+                </div>
+
+                <div class="column">
+                    <h3 class="title is-4">Feeds</h3>
+
+                    <div class="channel">
+                        {{ range .Feeds }}
+                            <div class="feed box">
+                                <div class="name">
+                                    <a href="{{ .URL }}">{{ .URL }}</a>
+                                </div>
+                            </div>
+                        {{ else }}
+                            <div class="no-channels">No feeds</div>
+                        {{ end }}
+                    </div>
+                </div>
+            </div>
+        </div>
+    </section>
+</body>
+</html>

templates/index.html

@@ -0,0 +1,71 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Ekster</title>
+<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.1/css/bulma.min.css">
+<link rel="micropub" href="{{ .Baseurl }}/micropub" />
+<link rel="authorization_endpoint" href="{{ .Baseurl }}/auth" />
+<link rel="token_endpoint" href="{{ .Baseurl }}/auth/token" />
+</head>
+<body>
+    <section class="section">
+        <div class="container">
+
+
+            <nav class="navbar" role="navigation" aria-label="main navigation">
+                <div class="navbar-brand">
+                    <a class="navbar-item" href="/">
+                        Ekster
+                    </a>
+
+                    <a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false" data-target="menu">
+                        <span aria-hidden="true"></span>
+                        <span aria-hidden="true"></span>
+                        <span aria-hidden="true"></span>
+                    </a>
+                </div>
+
+                {{ if .Session.LoggedIn }}
+                    <div id="menu" class="navbar-menu">
+                        <a class="navbar-item" href="/settings">
+                            Settings
+                        </a>
+                        <a class="navbar-item" href="/logs">
+                            Logs
+                        </a>
+                        <a class="navbar-item" href="{{ .Session.Me }}">
+                            Profile
+                        </a>
+                    </div>
+                {{ end }}
+            </nav>
+
+            <h1 class="title">Ekster - Microsub server</h1>
+
+            {{ if .Session.LoggedIn }}
+                <h2 class="title">Logout</h2>
+                <form action="/session/logout" method="post">
+                    <button type="submit" class="button is-info">Logout</button>
+                </form>
+            {{ else }}
+                <h2 class="title">Sign in to Ekster</h2>
+                <form action="/session" method="post">
+                    <div class="field">
+                        <label class="label" for="url"></label>
+                        <div class="control">
+                            <input type="text" name="url" id="url" class="input" placeholder="https://example.com/">
+                        </div>
+                    </div>
+                    <div class="field is-grouped">
+                        <div class="control">
+                            <button type="submit" class="button is-info">Login</button>
+                        </div>
+                    </div>
+                </form>
+            {{ end }}
+        </div>
+    </section>
+</body>
+</html>

templates/logs.html

@@ -0,0 +1,50 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Ekster</title>
+<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.1/css/bulma.min.css">
+</head>
+<body>
+    <section class="section">
+        <div class="container">
+
+
+            <nav class="navbar" role="navigation" aria-label="main navigation">
+                <div class="navbar-brand">
+                    <a class="navbar-item" href="/">
+                        Ekster
+                    </a>
+
+                    <a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false" data-target="menu">
+                        <span aria-hidden="true"></span>
+                        <span aria-hidden="true"></span>
+                        <span aria-hidden="true"></span>
+                    </a>
+                </div>
+
+                {{ if .Session.LoggedIn }}
+                    <div id="menu" class="navbar-menu">
+                        <a class="navbar-item" href="/settings">
+                            Settings
+                        </a>
+                        <a class="navbar-item" href="/logs">
+                            Logs
+                        </a>
+                        <a class="navbar-item" href="{{ .Session.Me }}">
+                            Profile
+                        </a>
+                    </div>
+                {{ end }}
+            </nav>
+
+            <h1 class="title">Ekster - Microsub server</h1>
+
+            <h2 class="subtitle">Logs</h2>
+
+            <p>Logs</p>
+        </div>
+    </section>
+</body>
+</html>

templates/settings.html

@@ -0,0 +1,63 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Ekster</title>
+<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.1/css/bulma.min.css">
+</head>
+<body>
+    <section class="section">
+        <div class="container">
+            <nav class="navbar" role="navigation" aria-label="main navigation">
+                <div class="navbar-brand">
+                    <a class="navbar-item" href="/">
+                        Ekster
+                    </a>
+
+                    <a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false" data-target="menu">
+                        <span aria-hidden="true"></span>
+                        <span aria-hidden="true"></span>
+                        <span aria-hidden="true"></span>
+                    </a>
+                </div>
+
+                {{ if .Session.LoggedIn }}
+                    <div id="menu" class="navbar-menu">
+                        <a class="navbar-item" href="/settings">
+                            Settings
+                        </a>
+                        <a class="navbar-item" href="/logs">
+                            Logs
+                        </a>
+                        <a class="navbar-item" href="{{ .Session.Me }}">
+                            Profile
+                        </a>
+                    </div>
+                {{ end }}
+            </nav>
+
+            <h1 class="title">Ekster - Microsub server</h1>
+
+            {{ if .Session.LoggedIn }}
+            {{ end }}
+
+            <h2 class="subtitle">Channels</h2>
+
+            <div class="channels">
+                {{ range .Channels }}
+                    <div class="channel box">
+                        <div class="name">
+                            <a href="/settings/channel?uid={{ .UID }}">
+                                {{ .Name }}
+                            </a>
+                        </div>
+                    </div>
+                {{ else }}
+                    <div class="no-channels">No channels</div>
+                {{ end }}
+            </div>
+        </div>
+    </section>
+</body>
+</html>