peter/ekster
1
0
Fork 0
Code Issues 3 Pull Requests Projects 3 Releases 1 Wiki Activity

Compare commits

base: peter:master
Branches Tags
peter:master
peter:wip-jwt-channels
peter:cleanup
peter:1.0.0-rc.1
peter:0.8.4
peter:0.8.3
peter:0.8.2
peter:0.8.0
peter:0.4.0
peter:0.3.0
..
compare: peter:cleanup
Branches Tags
peter:master
peter:wip-jwt-channels
peter:cleanup
peter:1.0.0-rc.1
peter:0.8.4
peter:0.8.3
peter:0.8.2
peter:0.8.0
peter:0.4.0
peter:0.3.0

5 Commits
master ... cleanup

Author SHA1 Message Date
Peter Stuifzand
a1be6f4e35
Extract fetchAlternateFeed
Some checks failed
continuous-integration/drone/push Build is passing
Details
continuous-integration/drone/pr Build is failing
Details
2021-08-01 21:44:09 +02:00
Peter Stuifzand
01b255b3f7
Cleanup Search(...) 
- Extract findFeeds
 2021-08-01 21:38:40 +02:00
Peter Stuifzand
25bdf5a4a2
Add type info to templates for Goland 2021-07-31 16:40:40 +02:00
Peter Stuifzand
c7f231a38e
Return error on channel page 2021-07-31 16:40:21 +02:00
Peter Stuifzand
dcc9bfa889
cleanup channel settings page backend 2021-07-31 16:23:56 +02:00
81 changed files with 1631 additions and 4080 deletions
Show Stats Expand all files Collapse all files

70
.drone.yml
View File

@ -1,74 +1,25 @@
---
kind: pipeline
type: docker
name: build and test
name: default
workspace:
base: /go
path: src/p83.nl/go/ekster
trigger:
event:
- push
services:
- name: redis
image: redis:5
- name: database
image: postgres:14
environment:
POSTGRES_DB: ekster_testing
POSTGRES_USER: postgres
POSTGRES_PASSWORD: simple
POSTGRES_HOST_AUTH_METHOD: trust
steps:
- name: testing
image: golang:1.18-alpine
image: golang:1.16-alpine
environment:
CGO_ENABLED: 0
GOOS: linux
GOARCH: amd64
commands:
- go version
- apk --no-cache add git
- go get -d -t ./...
- go build -buildvcs=false p83.nl/go/ekster/cmd/eksterd
- go vet ./...
- go test -v ./...
---
kind: pipeline
type: docker
name: move to production
workspace:
base: /go
path: src/p83.nl/go/ekster
trigger:
event:
- promote
target:
- production
steps:
- name: build
image: golang:1.18-alpine
environment:
CGO_ENABLED: 0
GOOS: linux
GOARCH: amd64
commands:
- go version
- apk --no-cache add git
- go get -d -t ./...
- go build -buildvcs=false p83.nl/go/ekster/cmd/eksterd
- go build p83.nl/go/ekster/cmd/eksterd
- go test ./...
- name: publish-personal
image: plugins/docker
depends_on:
- build
settings:
repo: registry.stuifzandapp.com/microsub-server
registry: registry.stuifzandapp.com
@ -77,10 +28,19 @@ steps:
password:
from_secret: docker_password
- name: publish-docker
image: plugins/docker
settings:
repo: pstuifzand/ekster
tags:
- alpine
username:
from_secret: docker_official_username
password:
from_secret: docker_official_password
- name: deploy
image: appleboy/drone-ssh
depends_on:
- publish-personal
settings:
host: microsub.stuifzandapp.com
username: microsub

25
.github/workflows/go.yml vendored
View File

@ -1,25 +0,0 @@
name: Go
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.16
- name: Build
run: go build -v ./...
- name: Test
run: go test -v ./...

1
.pre-commit-config.yaml
View File

@ -7,7 +7,6 @@ repos:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-yaml
args: [--allow-multiple-documents]
- id: check-merge-conflict
- id: check-added-large-files
- repo: https://github.com/dnephin/pre-commit-golang

26
CHANGELOG.md
View File

@ -1,26 +0,0 @@
# Changelog
## [Unreleased]
## [1.0.0-rc.1] - 2021-11-20
### Added
- Postgresql support for channels, feeds, items and subscriptions.
- Support "source" in items and feeds
### Changed
- Default channel backend is postgresql
### Fixed
- All `staticcheck` problems are fixed.
- Dependency between memorybackend and hubbackend removed and simplified.
### Deprecated
- All Redis timeline types are deprecated and will be removed in a later version.
[Unreleased]: https://git.p83.nl/peter/ekster/compare/1.0.0-rc.1...master
[1.0.0-rc.1]: https://git.p83.nl/peter/ekster/src/tag/1.0.0-rc.1

1
Dockerfile
View File

@ -3,4 +3,5 @@ RUN apk --no-cache add ca-certificates
WORKDIR /opt/micropub
EXPOSE 80
COPY ./eksterd /app/
COPY ./templates /app/templates
ENTRYPOINT ["/app/eksterd"]

3
README.md
View File

@ -140,8 +140,6 @@ support microsub.
-verbose
show verbose logging
Instead of the `UID` you can also use the `NAME` of the channel in most functions.
## Configuration: backend.json
The `backend.json` file contains all information about channels, feeds and authentication.
@ -155,6 +153,7 @@ The two parts that should be changed are:
"Me": "...",
"TokenEndpoint": "...",
The `Me` value should be set to the URL you use to sign into Monocle, or
Micropub client.

56
cmd/ek/main.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
// Ek is a microsub client.
package main
@ -144,7 +126,7 @@ Usage:
Commands:
connect URL login to a website that supports Indieauth and Microsub
connect URL login to Indieauth URL, e.g. your website
channels list channels
channels NAME create channel with NAME
@ -236,26 +218,6 @@ Global arguments:
performCommands(&c, flag.Args())
}
func channelID(sub microsub.Microsub, channelNameOrID string) (string, error) {
channels, err := sub.ChannelsGetList()
if err != nil {
// we encountered an error, so we are not sure if it worked
return channelNameOrID, err
}
for _, c := range channels {
if c.Name == channelNameOrID {
return c.UID, nil
}
if c.UID == channelNameOrID {
return c.UID, nil
}
}
// unknown?
return channelNameOrID, nil
}
func performCommands(sub microsub.Microsub, commands []string) {
if len(commands) == 0 {
flag.Usage()
@ -283,15 +245,15 @@ func performCommands(sub microsub.Microsub, commands []string) {
}
if len(commands) == 3 && commands[0] == "channels" {
if commands[1] == "-delete" {
uid, _ := channelID(sub, commands[2])
uid := commands[1]
if uid == "-delete" {
uid = commands[2]
err := sub.ChannelsDelete(uid)
if err != nil {
log.Fatalf("An error occurred: %s\n", err)
}
fmt.Printf("Channel %s deleted\n", uid)
} else {
uid, _ := channelID(sub, commands[1])
name := commands[2]
channel, err := sub.ChannelsUpdate(uid, name)
if err != nil {
@ -302,7 +264,7 @@ func performCommands(sub microsub.Microsub, commands []string) {
}
if len(commands) >= 2 && commands[0] == "timeline" {
channel, _ := channelID(sub, commands[1])
channel := commands[1]
var timeline microsub.Timeline
var err error
@ -342,7 +304,7 @@ func performCommands(sub microsub.Microsub, commands []string) {
query := commands[1]
var channel string
if len(commands) == 3 {
channel, _ = channelID(sub, commands[2])
channel = commands[2]
} else {
channel = "global"
}
@ -369,7 +331,7 @@ func performCommands(sub microsub.Microsub, commands []string) {
}
if len(commands) == 2 && commands[0] == "follow" {
uid, _ := channelID(sub, commands[1])
uid := commands[1]
feeds, err := sub.FollowGetList(uid)
if err != nil {
log.Fatalf("An error occurred: %s\n", err)
@ -380,7 +342,7 @@ func performCommands(sub microsub.Microsub, commands []string) {
}
if len(commands) == 3 && commands[0] == "follow" {
uid, _ := channelID(sub, commands[1])
uid := commands[1]
u := commands[2]
_, err := sub.FollowURL(uid, u)
if err != nil {
@ -390,7 +352,7 @@ func performCommands(sub microsub.Microsub, commands []string) {
}
if len(commands) == 3 && commands[0] == "unfollow" {
uid, _ := channelID(sub, commands[1])
uid := commands[1]
u := commands[2]
err := sub.UnfollowURL(uid, u)
if err != nil {

109
cmd/eksterd/app.go
View File

@ -1,109 +0,0 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package main
import (
"fmt"
"log"
"math/rand"
"net/http"
"time"
"github.com/pkg/errors"
"p83.nl/go/ekster/pkg/server"
)
func init() {
rand.Seed(time.Now().UnixNano())
}
// App is the main app structure
type App struct {
options AppOptions
backend *memoryBackend
hubBackend *hubIncomingBackend
}
// Run runs the app
func (app *App) Run() error {
err := initSearch()
if err != nil {
return fmt.Errorf("while starting app: %v", err)
}
app.backend.run()
app.hubBackend.run()
log.Printf("Listening on port %d\n", app.options.Port)
return http.ListenAndServe(fmt.Sprintf(":%d", app.options.Port), nil)
}
// NewApp initializes the App
func NewApp(options AppOptions) (*App, error) {
app := &App{
options: options,
}
backend, err := loadMemoryBackend(options.pool, options.database)
if err != nil {
return nil, err
}
app.backend = backend
// FIXME: load from database
app.backend.TokenEndpoint = "https://p83.nl/authtoken"
app.backend.Me = "https://p83.nl/"
app.backend.AuthEnabled = options.AuthEnabled
app.hubBackend = &hubIncomingBackend{
baseURL: options.BaseURL,
pool: options.pool,
database: options.database,
}
app.backend.hubBackend = app.hubBackend
http.Handle("/micropub", &micropubHandler{
Backend: app.backend,
pool: options.pool,
})
handler, broker := server.NewMicrosubHandler(app.backend)
if options.AuthEnabled {
handler = WithAuth(handler, app.backend)
}
app.backend.broker = broker
http.Handle("/microsub", handler)
http.Handle("/incoming/", &incomingHandler{
Backend: app.hubBackend,
Processor: app.backend,
})
if !options.Headless {
handler, err := newMainHandler(app.backend, options.BaseURL, options.TemplateDir, options.pool)
if err != nil {
return nil, errors.Wrap(err, "could not create main handler")
}
http.Handle("/", handler)
}
return app, nil
}

18
cmd/eksterd/auth.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package main
import (

123
cmd/eksterd/database_test.go
View File

@ -1,123 +0,0 @@
/*
* Ekster is a microsub server
* Copyright (c) 2022 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package main
import (
"database/sql"
"log"
"net/http/httptest"
"os"
"testing"
"github.com/gomodule/redigo/redis"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)
type DatabaseSuite struct {
suite.Suite
URL string
Database *sql.DB
RedisURL string
Redis redis.Conn
}
func (s *DatabaseSuite) SetupSuite() {
db, err := sql.Open("postgres", s.URL)
if err != nil {
log.Fatal(err)
}
s.Database = db
conn, err := redis.Dial("tcp", s.RedisURL)
if err != nil {
log.Fatal(err)
}
s.Redis = conn
_, err = s.Redis.Do("SELECT", "1")
if err != nil {
log.Fatal(err)
}
}
func (s *DatabaseSuite) TearDownSuite() {
err := s.Database.Close()
if err != nil {
log.Fatal(err)
}
err = s.Redis.Close()
if err != nil {
log.Fatal(err)
}
}
type databaseSuite struct {
DatabaseSuite
}
func (d *databaseSuite) TestGetChannelFromAuthorization() {
_, err := d.Database.Exec(`truncate "sources", "channels", "feeds", "subscriptions","items"`)
assert.NoError(d.T(), err, "truncate sources, channels, feeds")
row := d.Database.QueryRow(`INSERT INTO "channels" (uid, name, created_at, updated_at) VALUES ('abcdef', 'Channel', now(), now()) RETURNING "id"`)
var id int
err = row.Scan(&id)
assert.NoError(d.T(), err, "insert channel")
_, err = d.Database.Exec(`INSERT INTO "sources" (channel_id, auth_code, created_at, updated_at) VALUES ($1, '1234', now(), now())`, id)
assert.NoError(d.T(), err, "insert sources")
// source_id found
r := httptest.NewRequest("POST", "/micropub?source_id=1234", nil)
_, c, err := getChannelFromAuthorization(r, d.Redis, d.Database)
assert.NoError(d.T(), err, "channel from source_id")
assert.Equal(d.T(), "abcdef", c, "channel uid found")
// source_id not found
r = httptest.NewRequest("POST", "/micropub?source_id=1111", nil)
_, c, err = getChannelFromAuthorization(r, d.Redis, d.Database)
assert.Error(d.T(), err, "channel from authorization header")
assert.Equal(d.T(), "", c, "channel uid found")
}
func TestDatabaseSuite(t *testing.T) {
if testing.Short() {
t.Skip("Skip test for database")
}
databaseURL := os.Getenv("DATABASE_TEST_URL")
if databaseURL == "" {
databaseURL = "host=database user=postgres password=simple dbname=ekster_testing sslmode=disable"
}
databaseSuite := &databaseSuite{
DatabaseSuite{
URL: databaseURL,
RedisURL: "redis:6379",
},
}
databaseURL = "postgres://postgres@database/ekster_testing?sslmode=disable&user=postgres&password=simple"
err := runMigrations(databaseURL)
if err != nil {
log.Fatal(err)
}
suite.Run(t, databaseSuite)
}

19
cmd/eksterd/db/migrations/000001_create_channels_table.down.sql
View File

@ -1,19 +0,0 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
DROP TABLE "channels";

25
cmd/eksterd/db/migrations/000001_create_channels_table.up.sql
View File

@ -1,25 +0,0 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
CREATE TABLE IF NOT EXISTS "channels" (
"id" int primary key generated always as identity,
"uid" varchar(255) unique,
"name" varchar(255) unique,
"created_at" timestamptz DEFAULT current_timestamp,
"updated_at" timestamptz
);

19
cmd/eksterd/db/migrations/000002_create_feeds_table.down.sql
View File

@ -1,19 +0,0 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
DROP TABLE "feeds";

25
cmd/eksterd/db/migrations/000002_create_feeds_table.up.sql
View File

@ -1,25 +0,0 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
CREATE TABLE "feeds" (
"id" int primary key generated always as identity,
"channel_id" int references "channels" (id) on update cascade on delete cascade,
"url" varchar(512) not null unique,
"created_at" timestamptz DEFAULT current_timestamp,
"updated_at" timestamptz
);

19
cmd/eksterd/db/migrations/000003_add_default_channels.down.sql
View File

@ -1,19 +0,0 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
DELETE FROM "channels" WHERE "uid" IN ('home', 'notifications');

19
cmd/eksterd/db/migrations/000003_add_default_channels.up.sql
View File

@ -1,19 +0,0 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
INSERT INTO "channels" ("uid", "name") VALUES ('home', 'Home'), ('notifications', 'Notifications');

19
cmd/eksterd/db/migrations/000004_create_items_table.down.sql
View File

@ -1,19 +0,0 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
DROP TABLE "items";

28
cmd/eksterd/db/migrations/000004_create_items_table.up.sql
View File

@ -1,28 +0,0 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
CREATE TABLE IF NOT EXISTS "items" (
"id" int primary key generated always as identity,
"channel_id" int references "channels" on delete cascade,
"uid" varchar(512) not null unique,
"is_read" int default 0,
"data" jsonb,
"created_at" timestamptz DEFAULT current_timestamp,
"updated_at" timestamptz,
"published_at" timestamptz
);

19
cmd/eksterd/db/migrations/000005_alter_items_add_feed_id.down.sql
View File

@ -1,19 +0,0 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
ALTER TABLE "items" DROP COLUMN "feed_id";

19
cmd/eksterd/db/migrations/000005_alter_items_add_feed_id.up.sql
View File

@ -1,19 +0,0 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
ALTER TABLE "items" ADD COLUMN "feed_id" INT REFERENCES "feeds" ON DELETE CASCADE;

19
cmd/eksterd/db/migrations/000006_create_subscriptions_table.down.sql
View File

@ -1,19 +0,0 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
DROP TABLE "subscriptions";

30
cmd/eksterd/db/migrations/000006_create_subscriptions_table.up.sql
View File

@ -1,30 +0,0 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
CREATE TABLE "subscriptions" (
"id" int primary key generated always as identity,
"topic" varchar(1024) not null references "feeds" ("url") on update cascade on delete cascade,
"hub" varchar(1024) null,
"callback" varchar(1024) null,
"subscription_secret" varchar(32) not null,
"url_secret" varchar(32) not null,
"lease_seconds" int not null,
"created_at" timestamptz DEFAULT current_timestamp,
"updated_at" timestamptz,
"resubscribe_at" timestamptz
);

4
cmd/eksterd/db/migrations/000007_alter_table_add_feed_tier.down.sql
View File

@ -1,4 +0,0 @@
alter table "feeds"
drop column "tier",
drop column "unmodified",
drop column "next_fetch_at";

4
cmd/eksterd/db/migrations/000007_alter_table_add_feed_tier.up.sql
View File

@ -1,4 +0,0 @@
alter table "feeds"
add column "tier" int default 0,
add column "unmodified" int default 0,
add column "next_fetch_at" timestamptz;

1
cmd/eksterd/db/migrations/000008_create_table_sources.down.sql
View File

@ -1 +0,0 @@
DROP TABLE "sources";

20
cmd/eksterd/db/migrations/000008_create_table_sources.up.sql
View File

@ -1,20 +0,0 @@
BEGIN;
CREATE OR REPLACE FUNCTION update_timestamp()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = now();
RETURN NEW;
END;
$$ language 'plpgsql';
COMMIT;
CREATE TABLE "sources" (
"id" int primary key generated always as identity,
"channel_id" int not null,
"auth_code" varchar(64) not null,
"created_at" timestamp DEFAULT current_timestamp,
"updated_at" timestamp DEFAULT current_timestamp
);
CREATE TRIGGER sources_update_timestamp BEFORE INSERT OR UPDATE ON "sources"
FOR EACH ROW EXECUTE PROCEDURE update_timestamp();

124
cmd/eksterd/feedsearch.go Normal file
View File

@ -0,0 +1,124 @@
package main
import (
"fmt"
"log"
"net/http"
"net/url"
"strings"
"p83.nl/go/ekster/pkg/fetch"
"p83.nl/go/ekster/pkg/microsub"
"willnorris.com/go/microformats"
)
func isSupportedFeedType(feedType string) bool {
return strings.HasPrefix(feedType, "text/html") ||
strings.HasPrefix(feedType, "application/json") ||
strings.HasPrefix(feedType, "application/xml") ||
strings.HasPrefix(feedType, "text/xml") ||
strings.HasPrefix(feedType, "application/rss+xml") ||
strings.HasPrefix(feedType, "application/atom+xml")
}
func findFeeds(cachingFetch fetch.FetcherFunc, feedURL string) ([]microsub.Feed, error) {
resp, err := cachingFetch(feedURL)
if err != nil {
return nil, fmt.Errorf("while fetching %s: %w", feedURL, err)
}
defer resp.Body.Close()
fetchURL, err := url.Parse(feedURL)
md := microformats.Parse(resp.Body, fetchURL)
if err != nil {
return nil, fmt.Errorf("while fetching %s: %w", feedURL, err)
}
feedResp, err := cachingFetch(fetchURL.String())
if err != nil {
return nil, fmt.Errorf("in fetch of %s: %w", fetchURL, err)
}
defer feedResp.Body.Close()
// TODO: Combine FeedHeader and FeedItems so we can use it here
parsedFeed, err := fetch.FeedHeader(cachingFetch, fetchURL.String(), feedResp.Header.Get("Content-Type"), feedResp.Body)
if err != nil {
return nil, fmt.Errorf("in parse of %s: %w", fetchURL, err)
}
var feeds []microsub.Feed
// TODO: Only include the feed if it contains some items
feeds = append(feeds, parsedFeed)
// Fetch alternates
if alts, e := md.Rels["alternate"]; e {
for _, alt := range alts {
relURL := md.RelURLs[alt]
log.Printf("alternate found with type %s %#v\n", relURL.Type, relURL)
if isSupportedFeedType(relURL.Type) {
parsedFeed, err := fetchAlternateFeed(cachingFetch, alt)
if err != nil {
continue
}
feeds = append(feeds, parsedFeed)
}
}
}
return feeds, nil
}
func fetchAlternateFeed(cachingFetch fetch.FetcherFunc, altURL string) (microsub.Feed, error) {
feedResp, err := cachingFetch(altURL)
if err != nil {
return microsub.Feed{}, fmt.Errorf("fetch of %s: %v", altURL, err)
}
defer feedResp.Body.Close()
parsedFeed, err := fetch.FeedHeader(cachingFetch, altURL, feedResp.Header.Get("Content-Type"), feedResp.Body)
if err != nil {
return microsub.Feed{}, fmt.Errorf("in parse of %s: %v", altURL, err)
}
return parsedFeed, nil
}
func getPossibleURLs(query string) []string {
urls := []string{}
if !(strings.HasPrefix(query, "https://") || strings.HasPrefix(query, "http://")) {
secureURL := "https://" + query
if checkURL(secureURL) {
urls = append(urls, secureURL)
} else {
unsecureURL := "http://" + query
if checkURL(unsecureURL) {
urls = append(urls, unsecureURL)
}
}
} else {
urls = append(urls, query)
}
return urls
}
func checkURL(u string) bool {
testURL, err := url.Parse(u)
if err != nil {
return false
}
resp, err := http.Head(testURL.String())
if err != nil {
log.Printf("Error while HEAD %s: %v\n", u, err)
return false
}
defer resp.Body.Close()
return resp.StatusCode == 200
}

186
cmd/eksterd/http.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package main
import (
@ -124,6 +106,10 @@ func newMainHandler(backend *memoryBackend, baseURL, templateDir string, pool *r
return h, nil
}
func (h *mainHandler) templateFile(filename string) string {
return fmt.Sprintf("%s/%s", h.TemplateDir, filename)
}
func (h *mainHandler) renderTemplate(w io.Writer, filename string, data interface{}) error {
fsys, err := fs.Sub(templates, "templates")
if err != nil {
@ -306,7 +292,7 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
err := r.ParseForm()
if err != nil {
log.Println(err)
http.Error(w, fmt.Sprintf("Bad Request: %s", err.Error()), http.StatusBadRequest)
http.Error(w, fmt.Sprintf("Bad Request: %s", err.Error()), 400)
return
}
@ -331,19 +317,15 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} else if r.URL.Path == "/session/callback" {
c, err := r.Cookie("session")
if err == http.ErrNoCookie {
http.Redirect(w, r, "/", http.StatusFound)
http.Redirect(w, r, "/", 302)
return
} else if err != nil {
http.Error(w, "could not read cookie", http.StatusInternalServerError)
http.Error(w, "could not read cookie", 500)
return
}
sessionVar := c.Value
sess, err := loadSession(sessionVar, conn)
if err != nil {
fmt.Fprintf(w, "ERROR: %q\n", err)
return
}
verified, authResponse, err := performIndieauthCallback(h.BaseURL, r, &sess)
if err != nil {
@ -356,9 +338,9 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
saveSession(sessionVar, &sess, conn)
log.Printf("SESSION: %#v\n", sess)
if sess.NextURI != "" {
http.Redirect(w, r, sess.NextURI, http.StatusFound)
http.Redirect(w, r, sess.NextURI, 302)
} else {
http.Redirect(w, r, "/", http.StatusFound)
http.Redirect(w, r, "/", 302)
}
return
}
@ -366,16 +348,11 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} else if r.URL.Path == "/settings/channel" {
c, err := r.Cookie("session")
if err == http.ErrNoCookie {
http.Redirect(w, r, "/", http.StatusFound)
http.Redirect(w, r, "/", 302)
return
}
sessionVar := c.Value
sess, err := loadSession(sessionVar, conn)
if err != nil {
log.Printf("ERROR: %s\n", err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
if !isLoggedIn(h.Backend, &sess) {
w.WriteHeader(401)
@ -385,30 +362,35 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
var page settingsPage
page.Session = sess
currentChannel := r.URL.Query().Get("uid")
currentChannelUID := r.URL.Query().Get("uid")
page.Channels, err = h.Backend.ChannelsGetList()
if err != nil {
log.Printf("ERROR: %s\n", err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
page.Feeds, err = h.Backend.FollowGetList(currentChannel)
if err != nil {
log.Printf("ERROR: %s\n", err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
page.Feeds, err = h.Backend.FollowGetList(currentChannelUID)
var selectedChannel microsub.Channel
found := false
for _, v := range page.Channels {
if v.UID == currentChannel {
page.CurrentChannel = v
if setting, e := h.Backend.Settings[v.UID]; e {
if v.UID == currentChannelUID {
selectedChannel = v
found = true
break
}
}
if found {
page.CurrentChannel = selectedChannel
if setting, e := h.Backend.Settings[selectedChannel.UID]; e {
page.CurrentSetting = setting
} else {
page.CurrentSetting = channelSetting{}
}
// FIXME: similar code is found in timeline.go
if page.CurrentSetting.ChannelType == "" {
page.CurrentSetting.ChannelType = "postgres-stream"
if selectedChannel.UID == "notifications" {
page.CurrentSetting.ChannelType = "stream"
} else {
page.CurrentSetting.ChannelType = "sorted-set"
}
}
page.ExcludedTypeNames = map[string]string{
"repost": "Reposts",
@ -425,28 +407,21 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
for _, v := range page.CurrentSetting.ExcludeType {
page.ExcludedTypes[v] = true
}
break
}
}
err = h.renderTemplate(w, "channel.html", page)
if err != nil {
fmt.Fprintf(w, "ERROR: %s\n", err)
http.Error(w, err.Error(), 500)
}
return
} else if r.URL.Path == "/logs" {
c, err := r.Cookie("session")
if err == http.ErrNoCookie {
http.Redirect(w, r, "/", http.StatusFound)
http.Redirect(w, r, "/", 302)
return
}
sessionVar := c.Value
sess, err := loadSession(sessionVar, conn)
if err != nil {
log.Printf("ERROR: %s\n", err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
if !isLoggedIn(h.Backend, &sess) {
w.WriteHeader(401)
@ -465,16 +440,11 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} else if r.URL.Path == "/settings" {
c, err := r.Cookie("session")
if err == http.ErrNoCookie {
http.Redirect(w, r, "/", http.StatusFound)
http.Redirect(w, r, "/", 302)
return
}
sessionVar := c.Value
sess, err := loadSession(sessionVar, conn)
if err != nil {
log.Printf("ERROR: %s\n", err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
if !isLoggedIn(h.Backend, &sess) {
w.WriteHeader(401)
@ -485,11 +455,6 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
var page settingsPage
page.Session = sess
page.Channels, err = h.Backend.ChannelsGetList()
if err != nil {
log.Printf("ERROR: %s\n", err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
// page.Feeds = h.Backend.Feeds
err = h.renderTemplate(w, "settings.html", page)
@ -504,16 +469,11 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
sessionVar := getSessionCookie(w, r)
sess, err := loadSession(sessionVar, conn)
if err != nil {
log.Printf("ERROR: %s\n", err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
if !isLoggedIn(h.Backend, &sess) {
sess.NextURI = r.URL.String()
saveSession(sessionVar, &sess, conn)
http.Redirect(w, r, "/", http.StatusFound)
http.Redirect(w, r, "/", 302)
return
}
@ -555,11 +515,6 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
page.Scope = scope
page.State = state
page.Channels, err = h.Backend.ChannelsGetList()
if err != nil {
log.Printf("ERROR: %s\n", err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
app, err := getAppInfo(clientID)
if err != nil {
@ -577,7 +532,7 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/session" {
c, err := r.Cookie("session")
if err == http.ErrNoCookie {
http.Redirect(w, r, "/", http.StatusFound)
http.Redirect(w, r, "/", 302)
return
}
@ -588,7 +543,7 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
endpoints, err := getEndpoints(me)
if err != nil {
http.Error(w, fmt.Sprintf("Bad Request: %s, %s", err.Error(), me), http.StatusBadRequest)
http.Error(w, fmt.Sprintf("Bad Request: %s, %s", err.Error(), me), 400)
return
}
@ -598,7 +553,7 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
sess, err := loadSession(sessionVar, conn)
if err != nil {
http.Redirect(w, r, "/", http.StatusFound)
http.Redirect(w, r, "/", 302)
return
}
@ -610,12 +565,12 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
err = saveSession(sessionVar, &sess, conn)
if err != nil {
http.Redirect(w, r, "/", http.StatusFound)
http.Redirect(w, r, "/", 302)
return
}
authenticationURL := indieauth.CreateAuthenticationURL(*endpoints.AuthorizationEndpoint, endpoints.Me.String(), h.BaseURL, redirectURI, state)
http.Redirect(w, r, authenticationURL, http.StatusFound)
http.Redirect(w, r, authenticationURL, 302)
return
} else if r.URL.Path == "/session/logout" {
@ -668,7 +623,7 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
redirectURI.RawQuery = q.Encode()
log.Println(redirectURI)
http.Redirect(w, r, redirectURI.String(), http.StatusFound)
http.Redirect(w, r, redirectURI.String(), 302)
return
} else if r.URL.Path == "/auth/token" {
grantType := r.FormValue("grant_type")
@ -711,40 +666,45 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
w.Header().Add("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(&res); err != nil {
enc := json.NewEncoder(w)
err = enc.Encode(&res)
if err != nil {
log.Println(err)
fmt.Fprintf(w, "ERROR: %q", err)
return
}
return
} else if r.URL.Path == "/settings/channel" {
// defer h.Backend.save()
// uid := r.FormValue("uid")
//
// if h.Backend.Settings == nil {
// h.Backend.Settings = make(map[string]channelSetting)
// }
//
// excludeRegex := r.FormValue("exclude_regex")
// includeRegex := r.FormValue("include_regex")
// channelType := r.FormValue("type")
//
// setting, e := h.Backend.Settings[uid]
// if !e {
// setting = channelSetting{}
// }
// setting.ExcludeRegex = excludeRegex
// setting.IncludeRegex = includeRegex
// setting.ChannelType = channelType
// if values, e := r.Form["exclude_type"]; e {
// setting.ExcludeType = values
// }
// h.Backend.Settings[uid] = setting
defer h.Backend.save()
uid := r.FormValue("uid")
http.Redirect(w, r, "/settings", http.StatusFound)
if h.Backend.Settings == nil {
h.Backend.Settings = make(map[string]channelSetting)
}
excludeRegex := r.FormValue("exclude_regex")
includeRegex := r.FormValue("include_regex")
channelType := r.FormValue("type")
setting, e := h.Backend.Settings[uid]
if !e {
setting = channelSetting{}
}
setting.ExcludeRegex = excludeRegex
setting.IncludeRegex = includeRegex
setting.ChannelType = channelType
if values, e := r.Form["exclude_type"]; e {
setting.ExcludeType = values
}
h.Backend.Settings[uid] = setting
h.Backend.Debug()
http.Redirect(w, r, "/settings", 302)
return
} else if r.URL.Path == "/refresh" {
h.Backend.RefreshFeeds()
http.Redirect(w, r, "/", http.StatusFound)
http.Redirect(w, r, "/", 302)
return
}
}
@ -755,14 +715,14 @@ func (h *mainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
func httpSessionLogout(r *http.Request, w http.ResponseWriter, conn redis.Conn) {
c, err := r.Cookie("session")
if err == http.ErrNoCookie {
http.Redirect(w, r, "/", http.StatusFound)
http.Redirect(w, r, "/", 302)
return
}
if err == nil {
sessionVar := c.Value
_, _ = conn.Do("DEL", "session:"+sessionVar)
}
http.Redirect(w, r, "/", http.StatusFound)
http.Redirect(w, r, "/", 302)
}
type parsedEndpoints struct {

244
cmd/eksterd/hubbackend.go
View File

@ -1,32 +1,17 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package main
import (
"database/sql"
"expvar"
"fmt"
"io"
"log"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/pkg/errors"
"p83.nl/go/ekster/pkg/util"
"p83.nl/go/ekster/pkg/websub"
@ -38,29 +23,31 @@ const LeaseSeconds = 24 * 60 * 60
// HubBackend handles information for the incoming handler
type HubBackend interface {
GetFeeds() []Feed // Deprecated
Feeds() ([]Feed, error)
CreateFeed(url string) (int64, error)
CreateFeed(url, channel string) (int64, error)
GetSecret(feedID int64) string
UpdateFeed(processor ContentProcessor, feedID int64, contentType string, body io.Reader) error
UpdateFeed(feedID int64, contentType string, body io.Reader) error
FeedSetLeaseSeconds(feedID int64, leaseSeconds int64) error
Subscribe(feed *Feed) error
}
type hubIncomingBackend struct {
backend *memoryBackend
baseURL string
pool *redis.Pool
database *sql.DB
}
// Feed contains information about the feed subscriptions
type Feed struct {
ID int64
URL string
Callback string
Hub string
Secret string
LeaseSeconds int64
ResubscribeAt *time.Time
ID int64 `redis:"id"`
Channel string `redis:"channel"`
URL string `redis:"url"`
Callback string `redis:"callback"`
Hub string `redis:"hub"`
Secret string `redis:"secret"`
LeaseSeconds int64 `redis:"lease_seconds"`
ResubscribeAt int64 `redis:"resubscribe_at"`
}
var (
@ -72,35 +59,29 @@ func init() {
}
func (h *hubIncomingBackend) GetSecret(id int64) string {
db := h.database
var secret string
err := db.QueryRow(
`select "subscription_secret" from "subscriptions" where "id" = $1`,
id,
).Scan(&secret)
conn := h.pool.Get()
defer conn.Close()
secret, err := redis.String(conn.Do("HGET", fmt.Sprintf("feed:%d", id), "secret"))
if err != nil {
return ""
}
return secret
}
func (h *hubIncomingBackend) CreateFeed(topic string) (int64, error) {
log.Println("CreateFeed", topic)
db := h.database
func (h *hubIncomingBackend) CreateFeed(topic string, channel string) (int64, error) {
conn := h.pool.Get()
defer conn.Close()
secret := util.RandStringBytes(32)
urlSecret := util.RandStringBytes(32)
var subscriptionID int
err := db.QueryRow(`
INSERT INTO "subscriptions" ("topic","subscription_secret", "url_secret", "lease_seconds", "created_at")
VALUES ($1, $2, $3, $4, DEFAULT) RETURNING "id"`, topic, secret, urlSecret, 60*60*24*7).Scan(&subscriptionID)
// TODO(peter): check if topic already is registered
id, err := redis.Int64(conn.Do("INCR", "feed:next_id"))
if err != nil {
return 0, err
}
if err != nil {
return 0, fmt.Errorf("insert into subscriptions: %w", err)
}
conn.Do("HSET", fmt.Sprintf("feed:%d", id), "url", topic)
conn.Do("HSET", fmt.Sprintf("feed:%d", id), "channel", channel)
secret := util.RandStringBytes(16)
conn.Do("HSET", fmt.Sprintf("feed:%d", id), "secret", secret)
client := &http.Client{}
@ -110,107 +91,135 @@ VALUES ($1, $2, $3, $4, DEFAULT) RETURNING "id"`, topic, secret, urlSecret, 60*6
return 0, err
}
callbackURL := fmt.Sprintf("%s/incoming/%d", h.baseURL, subscriptionID)
callbackURL := fmt.Sprintf("%s/incoming/%d", h.baseURL, id)
log.Printf("WebSub Hub URL found for topic=%q hub=%q callback=%q\n", topic, hubURL, callbackURL)
if err == nil && hubURL != "" {
_, err := db.Exec(`UPDATE subscriptions SET hub = $1, callback = $2 WHERE id = $3`, hubURL, callbackURL, subscriptionID)
args := redis.Args{}.Add(fmt.Sprintf("feed:%d", id), "hub", hubURL, "callback", callbackURL)
_, err = conn.Do("HMSET", args...)
if err != nil {
return 0, fmt.Errorf("save hub and callback: %w", err)
return 0, errors.Wrap(err, "could not write to redis backend")
}
} else {
return int64(subscriptionID), nil
return id, nil
}
err = websub.Subscribe(client, hubURL, topic, callbackURL, secret, 24*3600)
if err != nil {
return 0, fmt.Errorf("subscribe: %w", err)
return 0, err
}
return int64(subscriptionID), nil
return id, nil
}
func (h *hubIncomingBackend) UpdateFeed(processor ContentProcessor, subscriptionID int64, contentType string, body io.Reader) error {
log.Println("UpdateFeed", subscriptionID)
db := h.database
// Process all channels that contains this feed
rows, err := db.Query(`
select topic, c.uid, f.id, c.name
from subscriptions s
inner join feeds f on f.url = s.topic
inner join channels c on c.id = f.channel_id
where s.id = $1
`,
subscriptionID,
)
func (h *hubIncomingBackend) UpdateFeed(feedID int64, contentType string, body io.Reader) error {
conn := h.pool.Get()
defer conn.Close()
log.Printf("updating feed %d", feedID)
u, err := redis.String(conn.Do("HGET", fmt.Sprintf("feed:%d", feedID), "url"))
if err != nil {
return err
}
channel, err := redis.String(conn.Do("HGET", fmt.Sprintf("feed:%d", feedID), "channel"))
if err != nil {
return err
}
for rows.Next() {
var topic, channel, feedID, channelName string
log.Printf("Updating feed %d - %s %s\n", feedID, u, channel)
err = h.backend.ProcessContent(channel, u, contentType, body)
if err != nil {
log.Printf("could not process content for channel %s: %s", channel, err)
}
err = rows.Scan(&topic, &channel, &feedID, &channelName)
return err
}
func (h *hubIncomingBackend) FeedSetLeaseSeconds(feedID int64, leaseSeconds int64) error {
conn := h.pool.Get()
defer conn.Close()
log.Printf("updating feed %d lease_seconds", feedID)
args := redis.Args{}.Add(fmt.Sprintf("feed:%d", feedID), "lease_seconds", leaseSeconds, "resubscribe_at", time.Now().Add(time.Duration(60*(leaseSeconds-15))*time.Second).Unix())
_, err := conn.Do("HMSET", args...)
if err != nil {
log.Println(err)
continue
return err
}
log.Printf("Updating feed %s %q in %q (%s)\n", feedID, topic, channelName, channel)
_, err = processor.ProcessContent(channel, feedID, topic, contentType, body)
return nil
}
// GetFeeds is deprecated, use Feeds instead
func (h *hubIncomingBackend) GetFeeds() []Feed {
log.Println("GetFeeds called, consider replacing with Feeds")
feeds, err := h.Feeds()
if err != nil {
log.Printf("could not process content for channel %s: %s", channelName, err)
log.Printf("Feeds returned an error: %v", err)
}
}
return err
}
func (h *hubIncomingBackend) FeedSetLeaseSeconds(subscriptionID int64, leaseSeconds int64) error {
db := h.database
_, err := db.Exec(`
update subscriptions
set lease_seconds = $1,
resubscribe_at = now() + $2 * interval '1' second
where id = $3
`, leaseSeconds, leaseSeconds, subscriptionID)
return err
return feeds
}
// Feeds returns a list of subscribed feeds
func (h *hubIncomingBackend) Feeds() ([]Feed, error) {
db := h.database
var feeds []Feed
conn := h.pool.Get()
defer conn.Close()
feeds := []Feed{}
rows, err := db.Query(`
select s.id, topic, hub, callback, subscription_secret, lease_seconds, resubscribe_at
from subscriptions s
inner join feeds f on f.url = s.topic
inner join channels c on c.id = f.channel_id
where hub is not null
`)
// FIXME(peter): replace with set of currently checked feeds
feedKeys, err := redis.Strings(conn.Do("KEYS", "feed:*"))
if err != nil {
return nil, err
return nil, errors.Wrap(err, "could not get feeds from backend")
}
for rows.Next() {
for _, feedKey := range feedKeys {
var feed Feed
err = rows.Scan(
&feed.ID,
&feed.URL,
&feed.Hub,
&feed.Callback,
&feed.Secret,
&feed.LeaseSeconds,
&feed.ResubscribeAt,
)
values, err := redis.Values(conn.Do("HGETALL", feedKey))
if err != nil {
log.Println("Feeds: scan subscriptions:", err)
log.Printf("could not get feed info for key %s: %v", feedKey, err)
continue
}
err = redis.ScanStruct(values, &feed)
if err != nil {
log.Printf("could not scan struct for key %s: %v", feedKey, err)
continue
}
// Add feed id
if feed.ID == 0 {
parts := strings.Split(feedKey, ":")
if len(parts) == 2 {
feed.ID, _ = strconv.ParseInt(parts[1], 10, 64)
_, err = conn.Do("HSET", feedKey, "id", feed.ID)
if err != nil {
log.Printf("could not save id for %s: %v", feedKey, err)
}
}
}
// Fix the callback url
callbackURL, err := url.Parse(feed.Callback)
if err != nil || !callbackURL.IsAbs() {
if err != nil {
log.Printf("could not parse callback url %q: %v", callbackURL, err)
} else {
log.Printf("url is relative; replace with absolute url: %q", callbackURL)
}
feed.Callback = fmt.Sprintf("%s/incoming/%d", h.baseURL, feed.ID)
_, err = conn.Do("HSET", feedKey, "callback", feed.Callback)
if err != nil {
log.Printf("could not save id for %s: %v", feedKey, err)
}
}
// Skip feeds without a Hub
if feed.Hub == "" {
continue
}
log.Printf("Websub feed: %#v\n", feed)
feeds = append(feeds, feed)
}
@ -218,33 +227,28 @@ func (h *hubIncomingBackend) Feeds() ([]Feed, error) {
}
func (h *hubIncomingBackend) Subscribe(feed *Feed) error {
log.Println("Subscribe", feed.URL)
client := http.Client{}
return websub.Subscribe(&client, feed.Hub, feed.URL, feed.Callback, feed.Secret, LeaseSeconds)
}
func (h *hubIncomingBackend) run() error {
ticker := time.NewTicker(1 * time.Minute)
ticker := time.NewTicker(10 * time.Minute)
quit := make(chan struct{})
go func() {
for {
select {
case <-ticker.C:
log.Println("Getting feeds for WebSub started")
log.Println("Getting feeds for WebSub")
varWebsub.Add("runs", 1)
feeds, err := h.Feeds()
if err != nil {
log.Println("Feeds failed:", err)
log.Println("Getting feeds for WebSub completed")
continue
}
log.Printf("Found %d feeds", len(feeds))
for _, feed := range feeds {
log.Printf("Looking at %s\n", feed.URL)
if feed.ResubscribeAt != nil && time.Now().After(*feed.ResubscribeAt) {
if feed.ResubscribeAt == 0 || time.Now().After(time.Unix(feed.ResubscribeAt, 0)) {
if feed.Callback == "" {
feed.Callback = fmt.Sprintf("%s/incoming/%d", h.baseURL, feed.ID)
}
@ -257,8 +261,6 @@ func (h *hubIncomingBackend) run() error {
}
}
}
log.Println("Getting feeds for WebSub completed")
case <-quit:
ticker.Stop()
return

42
cmd/eksterd/incoming.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package main
import (
@ -32,7 +14,6 @@ import (
type incomingHandler struct {
Backend HubBackend
Processor ContentProcessor
}
var (
@ -68,14 +49,12 @@ func (h *incomingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
leaseSeconds, err := strconv.ParseInt(leaseStr, 10, 64)
if err != nil {
log.Printf("error in hub.lease_seconds format %q: %s", leaseSeconds, err)
http.Error(w, fmt.Sprintf("error in hub.lease_seconds format %q: %s", leaseSeconds, err), http.StatusBadRequest)
http.Error(w, fmt.Sprintf("error in hub.lease_seconds format %q: %s", leaseSeconds, err), 400)
return
}
err = h.Backend.FeedSetLeaseSeconds(feed, leaseSeconds)
if err != nil {
log.Printf("error in while setting hub.lease_seconds: %s", err)
http.Error(w, fmt.Sprintf("error in while setting hub.lease_seconds: %s", err), http.StatusBadRequest)
http.Error(w, fmt.Sprintf("error in while setting hub.lease_seconds: %s", err), 400)
return
}
}
@ -88,7 +67,7 @@ func (h *incomingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
if r.Method != http.MethodPost {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
http.Error(w, "Method not allowed", 405)
return
}
@ -96,31 +75,28 @@ func (h *incomingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
secret := h.Backend.GetSecret(feed)
if secret == "" {
log.Printf("missing secret for feed %d\n", feed)
http.Error(w, "Unknown", http.StatusBadRequest)
http.Error(w, "Unknown", 400)
return
}
feedContent, err := ioutil.ReadAll(r.Body)
if err != nil {
log.Printf("ERROR: %s\n", err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
// match signature
sig := r.Header.Get("X-Hub-Signature")
if sig != "" {
if err := websub.ValidateHubSignature(sig, feedContent, []byte(secret)); err != nil {
log.Printf("could not validate signature: %+v", err)
http.Error(w, fmt.Sprintf("could not validate signature: %s", err), http.StatusBadRequest)
http.Error(w, fmt.Sprintf("could not validate signature: %s", err), 400)
return
}
}
ct := r.Header.Get("Content-Type")
err = h.Backend.UpdateFeed(h.Processor, feed, ct, bytes.NewBuffer(feedContent))
err = h.Backend.UpdateFeed(feed, ct, bytes.NewBuffer(feedContent))
if err != nil {
http.Error(w, fmt.Sprintf("could not update feed: %s (%s)", ct, err), http.StatusBadRequest)
http.Error(w, fmt.Sprintf("could not update feed: %s (%s)", ct, err), 400)
return
}
return
}

195
cmd/eksterd/main.go
View File

@ -1,20 +1,14 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
// Copyright (C) 2018 Peter Stuifzand
//
// This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any
// later version.
//
// This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along with this program. If not,
// see <http://www.gnu.org/licenses/>.
/*
Eksterd is a microsub server that is extendable.
@ -23,20 +17,18 @@ package main
import (
"database/sql"
"embed"
_ "expvar"
"flag"
"fmt"
"log"
"net/http"
"os"
"time"
"github.com/gomodule/redigo/redis"
"github.com/pkg/errors"
"p83.nl/go/ekster/pkg/auth"
"github.com/golang-migrate/migrate/v4"
_ "github.com/golang-migrate/migrate/v4/database/postgres"
"github.com/golang-migrate/migrate/v4/source/iofs"
"github.com/gomodule/redigo/redis"
"p83.nl/go/ekster/pkg/server"
)
// AppOptions are options for the app
@ -51,9 +43,6 @@ type AppOptions struct {
database *sql.DB
}
//go:embed db/migrations/*.sql
var migrations embed.FS
func init() {
log.SetFlags(log.Lshortfile | log.Ldate | log.Ltime)
}
@ -92,13 +81,13 @@ func WithAuth(handler http.Handler, b *memoryBackend) http.Handler {
}
if !authorized {
log.Printf("Token could not be validated")
http.Error(w, "Can't validate token", http.StatusForbidden)
http.Error(w, "Can't validate token", 403)
return
}
if token.Me != b.Me { // FIXME: Me should be part of the request
log.Printf("Missing \"me\" in token response: %#v\n", token)
http.Error(w, "Wrong me", http.StatusForbidden)
http.Error(w, "Wrong me", 403)
return
}
@ -106,6 +95,73 @@ func WithAuth(handler http.Handler, b *memoryBackend) http.Handler {
})
}
// App is the main app structure
type App struct {
options AppOptions
backend *memoryBackend
hubBackend *hubIncomingBackend
}
// Run runs the app
func (app *App) Run() error {
err := initSearch()
if err != nil {
return fmt.Errorf("while starting app: %v", err)
}
app.backend.run()
app.hubBackend.run()
log.Printf("Listening on port %d\n", app.options.Port)
return http.ListenAndServe(fmt.Sprintf(":%d", app.options.Port), nil)
}
// NewApp initializes the App
func NewApp(options AppOptions) (*App, error) {
app := &App{
options: options,
}
backend, err := loadMemoryBackend(options.pool, options.database)
if err != nil {
return nil, err
}
app.backend = backend
app.backend.AuthEnabled = options.AuthEnabled
app.backend.baseURL = options.BaseURL
app.backend.hubIncomingBackend.pool = options.pool
app.backend.hubIncomingBackend.baseURL = options.BaseURL
app.hubBackend = &hubIncomingBackend{backend: app.backend, baseURL: options.BaseURL, pool: options.pool}
http.Handle("/micropub", &micropubHandler{
Backend: app.backend,
pool: options.pool,
})
handler, broker := server.NewMicrosubHandler(app.backend)
if options.AuthEnabled {
handler = WithAuth(handler, app.backend)
}
app.backend.broker = broker
http.Handle("/microsub", handler)
http.Handle("/incoming/", &incomingHandler{
Backend: app.hubBackend,
})
if !options.Headless {
handler, err := newMainHandler(app.backend, options.BaseURL, options.TemplateDir, options.pool)
if err != nil {
return nil, errors.Wrap(err, "could not create main handler")
}
http.Handle("/", handler)
}
return app, nil
}
func main() {
log.Println("eksterd - microsub server")
@ -141,36 +197,28 @@ func main() {
log.Fatal("EKSTER_TEMPLATES environment variable not found, use env var or -templates dir option")
}
}
//
// createBackend := false
// args := flag.Args()
//
// if len(args) >= 1 {
// if args[0] == "new" {
// createBackend = true
// }
// }
//
// if createBackend {
// err := createMemoryBackend()
// if err != nil {
// log.Fatalf("Error while saving backend.json: %s", err)
// }
//
// TODO(peter): automatically gather this information from login or otherwise
//
// log.Println(`Config file "backend.json" is created in the current directory.`)
// log.Println(`Update "Me" variable to your website address "https://example.com/"`)
// log.Println(`Update "TokenEndpoint" variable to the address of your token endpoint "https://example.com/token"`)
//
// return
// }
createBackend := false
args := flag.Args()
if len(args) >= 1 {
if args[0] == "new" {
createBackend = true
}
}
if createBackend {
err := createMemoryBackend()
if err != nil {
log.Fatalf("Error while saving backend.json: %s", err)
}
// TODO(peter): automatically gather this information from login or otherwise
databaseURL := "postgres://postgres@database/ekster?sslmode=disable&user=postgres&password=simple"
err := runMigrations(databaseURL)
if err != nil {
log.Fatalf("Error with migrations: %s", err)
log.Println(`Config file "backend.json" is created in the current directory.`)
log.Println(`Update "Me" variable to your website address "https://example.com/"`)
log.Println(`Update "TokenEndpoint" variable to the address of your token endpoint "https://example.com/token"`)
return
}
pool := newPool(options.RedisServer)
@ -190,38 +238,3 @@ func main() {
db.Close()
}
// Log migrations
type Log struct {
}
// Printf for migrations logs
func (l Log) Printf(format string, v ...interface{}) {
log.Printf(format, v...)
}
// Verbose returns false
func (l Log) Verbose() bool {
return false
}
func runMigrations(databaseURL string) error {
d, err := iofs.New(migrations, "db/migrations")
if err != nil {
return err
}
m, err := migrate.NewWithSourceInstance("iofs", d, databaseURL)
if err != nil {
return err
}
defer m.Close()
m.Log = &Log{}
log.Println("Running migrations")
if err = m.Up(); err != nil {
if err != migrate.ErrNoChange {
return err
}
}
log.Println("Migrations are up")
return nil
}

899
cmd/eksterd/memory.go
View File

File diff suppressed because it is too large Load Diff

338
cmd/eksterd/memory_test.go
View File

@ -1,228 +1,114 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package main
// func Test_memoryBackend_ChannelsCreate(t *testing.T) {
// type fields struct {
// hubIncomingBackend hubIncomingBackend
// lock sync.RWMutex
// Channels map[string]microsub.Channel
// Feeds map[string][]microsub.Feed
// Settings map[string]channelSetting
// NextUID int
// Me string
// TokenEndpoint string
// AuthEnabled bool
// ticker *time.Ticker
// quit chan struct{}
// broker *sse.Broker
// pool *redis.Pool
// }
// type args struct {
// name string
// }
// tests := []struct {
// name string
// fields fields
// args args
// want microsub.Channel
// wantErr bool
// }{
// {
// name: "Duplicate channel",
// fields: fields{
// hubIncomingBackend: hubIncomingBackend{},
// lock: sync.RWMutex{},
// Channels: func() map[string]microsub.Channel {
// channels := make(map[string]microsub.Channel)
// channels["1234"] = microsub.Channel{
// UID: "1234",
// Name: "Test",
// Unread: microsub.Unread{
// Type: microsub.UnreadCount,
// Unread: false,
// UnreadCount: 0,
// },
// }
// return channels
// }(),
// Feeds: func() map[string][]microsub.Feed {
// feeds := make(map[string][]microsub.Feed)
// return feeds
// }(),
// Settings: nil,
// NextUID: 1,
// Me: "",
// TokenEndpoint: "",
// AuthEnabled: false,
// ticker: nil,
// quit: nil,
// broker: nil,
// pool: nil,
// },
// args: args{
// name: "Test",
// },
// want: microsub.Channel{
// UID: "1234",
// Name: "Test",
// Unread: microsub.Unread{
// Type: microsub.UnreadCount,
// Unread: false,
// UnreadCount: 0,
// },
// },
// wantErr: false,
// },
// }
// for _, tt := range tests {
// t.Run(tt.name, func(t *testing.T) {
// b := &memoryBackend{
// hubIncomingBackend: tt.fields.hubIncomingBackend,
// lock: tt.fields.lock,
// Channels: tt.fields.Channels,
// Feeds: tt.fields.Feeds,
// Settings: tt.fields.Settings,
// NextUID: tt.fields.NextUID,
// Me: tt.fields.Me,
// TokenEndpoint: tt.fields.TokenEndpoint,
// AuthEnabled: tt.fields.AuthEnabled,
// ticker: tt.fields.ticker,
// quit: tt.fields.quit,
// broker: tt.fields.broker,
// pool: tt.fields.pool,
// }
// got, err := b.ChannelsCreate(tt.args.name)
// if (err != nil) != tt.wantErr {
// t.Errorf("ChannelsCreate() error = %v, wantErr %v", err, tt.wantErr)
// return
// }
// if !reflect.DeepEqual(got, tt.want) {
// t.Errorf("ChannelsCreate() got = %v, want %v", got, tt.want)
// }
// })
// }
// }
//
// func Test_memoryBackend_removeFeed(t *testing.T) {
// type fields struct {
// Channels map[string]microsub.Channel
// Feeds map[string][]microsub.Feed
// }
// type args struct {
// feedID string
// }
// tests := []struct {
// name string
// fields fields
// args args
// lens map[string]int
// wantErr bool
// }{
// {
// name: "remove from channel 1",
// fields: fields{
// Channels: map[string]microsub.Channel{
// "123": {UID: "channel1", Name: "Channel 1"},
// "124": {UID: "channel2", Name: "Channel 2"},
// },
// Feeds: map[string][]microsub.Feed{
// "123": {{Type: "feed", URL: "feed1", Name: "Feed1"}},
// "124": {{Type: "feed", URL: "feed2", Name: "Feed2"}},
// },
// },
// args: args{feedID: "feed1"},
// lens: map[string]int{"123": 0, "124": 1},
// wantErr: false,
// },
// {
// name: "remove from channel 2",
// fields: fields{
// Channels: map[string]microsub.Channel{
// "123": {UID: "channel1", Name: "Channel 1"},
// "124": {UID: "channel2", Name: "Channel 2"},
// },
// Feeds: map[string][]microsub.Feed{
// "123": {{Type: "feed", URL: "feed1", Name: "Feed1"}},
// "124": {{Type: "feed", URL: "feed2", Name: "Feed2"}},
// },
// },
// args: args{feedID: "feed2"},
// lens: map[string]int{"123": 1, "124": 0},
// wantErr: false,
// },
// {
// name: "remove unknown",
// fields: fields{
// Channels: map[string]microsub.Channel{
// "123": {UID: "channel1", Name: "Channel 1"},
// "124": {UID: "channel2", Name: "Channel 2"},
// },
// Feeds: map[string][]microsub.Feed{
// "123": {{Type: "feed", URL: "feed1", Name: "Feed1"}},
// "124": {{Type: "feed", URL: "feed2", Name: "Feed2"}},
// },
// },
// args: args{feedID: "feed3"},
// lens: map[string]int{"123": 1, "124": 1},
// wantErr: false,
// },
// {
// name: "remove from 0 channels",
// fields: fields{
// Channels: map[string]microsub.Channel{},
// Feeds: map[string][]microsub.Feed{},
// },
// args: args{feedID: "feed3"},
// lens: map[string]int{},
// wantErr: false,
// },
// {
// name: "remove from multiple channels",
// fields: fields{
// Channels: map[string]microsub.Channel{
// "123": {UID: "channel1", Name: "Channel 1"},
// "124": {UID: "channel2", Name: "Channel 2"},
// },
// Feeds: map[string][]microsub.Feed{
// "123": {{Type: "feed", URL: "feed1", Name: "Feed1"}},
// "124": {{Type: "feed", URL: "feed1", Name: "Feed1"}},
// },
// },
// args: args{feedID: "feed1"},
// lens: map[string]int{"123": 0, "124": 0},
// wantErr: false,
// },
// }
// for _, tt := range tests {
// t.Run(tt.name, func(t *testing.T) {
// b := &memoryBackend{
// Channels: tt.fields.Channels,
// Feeds: tt.fields.Feeds,
// }
// if err := b.removeFeed(tt.args.feedID); (err != nil) != tt.wantErr {
// t.Errorf("removeFeed() error = %v, wantErr %v", err, tt.wantErr)
// }
// assert.Len(t, b.Channels, len(tt.lens))
// for k, v := range tt.lens {
// assert.Len(t, b.Feeds[k], v)
// }
// })
// }
// }
import (
"reflect"
"sync"
"testing"
"time"
"github.com/gomodule/redigo/redis"
"p83.nl/go/ekster/pkg/microsub"
"p83.nl/go/ekster/pkg/sse"
)
func Test_memoryBackend_ChannelsCreate(t *testing.T) {
type fields struct {
hubIncomingBackend hubIncomingBackend
lock sync.RWMutex
Channels map[string]microsub.Channel
Feeds map[string][]microsub.Feed
Settings map[string]channelSetting
NextUID int
Me string
TokenEndpoint string
AuthEnabled bool
ticker *time.Ticker
quit chan struct{}
broker *sse.Broker
pool *redis.Pool
}
type args struct {
name string
}
tests := []struct {
name string
fields fields
args args
want microsub.Channel
wantErr bool
}{
{
name: "Duplicate channel",
fields: fields{
hubIncomingBackend: hubIncomingBackend{},
lock: sync.RWMutex{},
Channels: func() map[string]microsub.Channel {
channels := make(map[string]microsub.Channel)
channels["1234"] = microsub.Channel{
UID: "1234",
Name: "Test",
Unread: microsub.Unread{
Type: microsub.UnreadCount,
Unread: false,
UnreadCount: 0,
},
}
return channels
}(),
Feeds: func() map[string][]microsub.Feed {
feeds := make(map[string][]microsub.Feed)
return feeds
}(),
Settings: nil,
NextUID: 1,
Me: "",
TokenEndpoint: "",
AuthEnabled: false,
ticker: nil,
quit: nil,
broker: nil,
pool: nil,
},
args: args{
name: "Test",
},
want: microsub.Channel{
UID: "1234",
Name: "Test",
Unread: microsub.Unread{
Type: microsub.UnreadCount,
Unread: false,
UnreadCount: 0,
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
b := &memoryBackend{
hubIncomingBackend: tt.fields.hubIncomingBackend,
lock: tt.fields.lock,
Channels: tt.fields.Channels,
Feeds: tt.fields.Feeds,
Settings: tt.fields.Settings,
NextUID: tt.fields.NextUID,
Me: tt.fields.Me,
TokenEndpoint: tt.fields.TokenEndpoint,
AuthEnabled: tt.fields.AuthEnabled,
ticker: tt.fields.ticker,
quit: tt.fields.quit,
broker: tt.fields.broker,
pool: tt.fields.pool,
}
got, err := b.ChannelsCreate(tt.args.name)
if (err != nil) != tt.wantErr {
t.Errorf("ChannelsCreate() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("ChannelsCreate() got = %v, want %v", got, tt.want)
}
})
}
}

105
cmd/eksterd/micropub.go
View File

@ -1,26 +1,7 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package main
import (
"crypto/sha1"
"database/sql"
"encoding/json"
"fmt"
"log"
@ -59,57 +40,44 @@ func (h *micropubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
err := r.ParseForm()
if err != nil {
http.Error(w, "bad request", http.StatusBadRequest)
http.Error(w, "bad request", 400)
return
}
if r.Method == http.MethodPost {
var channel string
sourceID, channel, err := getChannelFromAuthorization(r, conn, h.Backend.database)
channel, err = getChannelFromAuthorization(r, conn)
if err != nil {
log.Println(err)
http.Error(w, "unauthorized", http.StatusUnauthorized)
http.Error(w, "unauthorized", 401)
return
}
// no channel is found
if channel == "" {
http.Error(w, "bad request, unknown channel", http.StatusBadRequest)
http.Error(w, "bad request, unknown channel", 400)
return
}
// TODO: We could try to fill the Source of the Item with something, but what?
item, err := parseIncomingItem(r)
if err != nil {
log.Println(err)
http.Error(w, err.Error(), http.StatusBadRequest)
http.Error(w, err.Error(), 400)
return
}
log.Printf("Item published: %s", item.Published)
if item.Published == "" {
item.Published = time.Now().Format(time.RFC3339)
}
item.Read = false
newID, err := generateItemID(conn, channel)
if err != nil {
log.Println(err)
http.Error(w, err.Error(), http.StatusInternalServerError)
http.Error(w, err.Error(), 500)
return
}
item.ID = newID
item.Source = &microsub.Source{
ID: fmt.Sprintf("micropub:%d", sourceID),
Name: fmt.Sprintf("Source %d", sourceID),
}
_, err = h.Backend.channelAddItemWithMatcher(channel, *item)
err = h.Backend.channelAddItemWithMatcher(channel, *item)
if err != nil {
log.Printf("could not add item to channel %s: %v", channel, err)
}
err = h.Backend.updateChannelUnreadCount(channel)
if err != nil {
log.Printf("could not update channel unread content %s: %v", channel, err)
@ -118,14 +86,12 @@ func (h *micropubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if err = json.NewEncoder(w).Encode(map[string]string{"ok": "1"}); err != nil {
log.Println(err)
http.Error(w, "internal server error", http.StatusInternalServerError)
http.Error(w, "internal server error", 500)
}
return
}
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
http.Error(w, "Method not allowed", 405)
}
func generateItemID(conn redis.Conn, channel string) (string, error) {
@ -137,57 +103,52 @@ func generateItemID(conn redis.Conn, channel string) (string, error) {
}
func parseIncomingItem(r *http.Request) (*microsub.Item, error) {
var item microsub.Item
contentType := r.Header.Get("content-type")
if contentType == "application/jf2+json" {
var item microsub.Item
if err := json.NewDecoder(r.Body).Decode(&item); err != nil {
return nil, fmt.Errorf("could not decode request body as %q: %v", contentType, err)
dec := json.NewDecoder(r.Body)
err := dec.Decode(&item)
if err != nil {
return nil, errors.Wrapf(err, "could not decode request body as jf2: %v", err)
}
return &item, nil
} else if contentType == "application/json" {
var mfItem microformats.Microformat
if err := json.NewDecoder(r.Body).Decode(&mfItem); err != nil {
return nil, fmt.Errorf("could not decode request body as %q: %v", contentType, err)
dec := json.NewDecoder(r.Body)
err := dec.Decode(&mfItem)
if err != nil {
return nil, errors.Wrapf(err, "could not decode request body as json: %v", err)
}
author := microsub.Card{}
item, ok := jf2.SimplifyMicroformatItem(&mfItem, author)
var ok bool
item, ok = jf2.SimplifyMicroformatItem(&mfItem, author)
if !ok {
return nil, fmt.Errorf("could not simplify microformat item to jf2")
}
return &item, nil
} else if contentType == "application/x-www-form-urlencoded" {
// TODO: improve handling of form-urlencoded
var item microsub.Item
content := r.FormValue("content")
name := r.FormValue("name")
item.Type = "entry"
item.Name = name
item.Content = &microsub.Content{Text: content}
item.Published = time.Now().Format(time.RFC3339)
} else {
return nil, fmt.Errorf("content-type %s is not supported", contentType)
}
return &item, nil
}
return nil, fmt.Errorf("content-type %q is not supported", contentType)
}
func getChannelFromAuthorization(r *http.Request, conn redis.Conn, database *sql.DB) (int, string, error) {
func getChannelFromAuthorization(r *http.Request, conn redis.Conn) (string, error) {
// backward compatible
sourceID := r.URL.Query().Get("source_id")
if sourceID != "" {
row := database.QueryRow(`
SELECT s.id as source_id, c.uid
FROM "sources" AS "s"
INNER JOIN "channels" AS "c" ON s.channel_id = c.id
WHERE "auth_code" = $1
`, sourceID)
var channel string
var sourceID int
if err := row.Scan(&sourceID, &channel); err == sql.ErrNoRows {
return 0, "", errors.New("channel not found")
channel, err := redis.String(conn.Do("HGET", "sources", sourceID))
if err != nil {
return "", errors.Wrapf(err, "could not get channel for sourceID: %s", sourceID)
}
return sourceID, channel, nil
return channel, nil
}
// full micropub with indieauth
@ -196,11 +157,11 @@ WHERE "auth_code" = $1
token := authHeader[7:]
channel, err := redis.String(conn.Do("HGET", "token:"+token, "channel"))
if err != nil {
return 0, "", errors.Wrap(err, "could not get channel for token")
return "", errors.Wrap(err, "could not get channel for token")
}
return 0, channel, nil
return channel, nil
}
return 0, "", fmt.Errorf("could not get channel from authorization")
return "", fmt.Errorf("could not get channel from authorization")
}

18
cmd/eksterd/search.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package main
import (

3
cmd/eksterd/templates/auth.html
View File

@ -1,5 +1,6 @@
{{- /*gotype: p83.nl/go/ekster/cmd/eksterd.authPage*/ -}}
<!DOCTYPE html>
<html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">

1
cmd/eksterd/templates/channel.html
View File

@ -1,3 +1,4 @@
{{- /*gotype: p83.nl/go/ekster/cmd/eksterd.settingsPage*/ -}}
<!DOCTYPE html>
<html>
<head>

7
cmd/eksterd/templates/index.html
View File

@ -12,6 +12,8 @@
<body>
<section class="section">
<div class="container">
<nav class="navbar" role="navigation" aria-label="main navigation">
<div class="navbar-brand">
<a class="navbar-item" href="/">
@ -63,11 +65,6 @@
</div>
</form>
{{ end }}
<p>Copyright (C) 2018 The Ekster authors.<br>
This program comes with ABSOLUTELY NO WARRANTY. This is free software,
and you are welcome to redistribute it under certain conditions.
See <a href="https://www.gnu.org/licenses/gpl-3.0.en.html">Licence</a> for more details.</p>
</div>
</section>
</body>

20
cmd/jf2test/main.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package main
import (
@ -50,7 +32,7 @@ func main() {
}
defer resp.Body.Close()
items, err := fetch.FeedItems(fetch.FetcherFunc(Fetch), url, resp.Header.Get("Content-Type"), resp.Body)
items, err := fetch.FeedItems(Fetch, url, resp.Header.Get("Content-Type"), resp.Body)
if err != nil {
log.Fatal(err)
}

15
docker-compose.yml
View File

@ -4,7 +4,7 @@ services:
image: "redis:5"
database:
image: postgres:14
image: postgres
volumes:
- database-data:/var/lib/postgresql/data
environment:
@ -14,16 +14,21 @@ services:
POSTGRES_HOST_AUTH_METHOD: trust
web:
image: ubuntu
working_dir: /app
image: "pstuifzand/ekster:alpine"
working_dir: /opt/microsub
links:
- redis:redis
volumes:
- microsub-data:/opt/microsub
- ./templates:/app/templates
- ./eksterd:/app/eksterd
- ./backend.json:/app/backend.json
entrypoint: /app/eksterd
command: -auth=false -port 80
command: -auth=false -port 80 -templates templates
ports:
- 8089:80
environment:
- "FEEDBIN_USER="
- "FEEDBIN_PASS="
- "EKSTER_BASEURL=http://localhost:8089/"
- "EKSTER_TEMPLATES=/app/templates"

9
go.mod
View File

@ -3,16 +3,15 @@ module p83.nl/go/ekster
go 1.16
require (
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751
github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394
github.com/blevesearch/bleve/v2 v2.0.3
github.com/blevesearch/bleve/v2 v2.0.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/gilliek/go-opml v1.0.0
github.com/golang-migrate/migrate/v4 v4.15.1
github.com/gomodule/redigo v1.8.2
github.com/lib/pq v1.10.1
github.com/pkg/errors v0.9.1
github.com/stretchr/testify v1.7.0
golang.org/x/net v0.0.0-20211013171255-e13a2654a71e
golang.org/x/text v0.3.7
github.com/stretchr/testify v1.5.1
golang.org/x/net v0.0.0-20200707034311-ab3426394381
willnorris.com/go/microformats v1.1.0
)

1450
go.sum
View File

File diff suppressed because it is too large Load Diff

20
pkg/auth/types.go
View File

@ -1,24 +1,6 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package auth
// Auther checks of the token in the head is accepted and fills TokenResponse
// Auther
type Auther interface {
AuthTokenAccepted(header string, r *TokenResponse) bool
}

24
pkg/client/requests.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package client
import (
@ -106,7 +88,7 @@ func (c *Client) microsubPostRequest(action string, args map[string]string) (*ht
if res.StatusCode != 200 {
msg, _ := ioutil.ReadAll(res.Body)
return nil, fmt.Errorf("unsuccessful response: %d: %q", res.StatusCode, strings.TrimSpace(string(msg)))
return nil, fmt.Errorf("unsuccessful response: %d: %q", res.StatusCode, string(msg))
}
return res, err
@ -134,7 +116,7 @@ func (c *Client) microsubPostFormRequest(action string, args map[string]string,
if res.StatusCode != 200 {
msg, _ := ioutil.ReadAll(res.Body)
return nil, fmt.Errorf("unsuccessful response: %d: %q", res.StatusCode, strings.TrimSpace(string(msg)))
return nil, fmt.Errorf("unsuccessful response: %d: %q", res.StatusCode, string(msg))
}
return res, err
@ -198,7 +180,7 @@ func (c *Client) TimelineGet(before, after, channel string) (microsub.Timeline,
func (c *Client) PreviewURL(url string) (microsub.Timeline, error) {
args := make(map[string]string)
args["url"] = url
res, err := c.microsubPostRequest("preview", args)
res, err := c.microsubGetRequest("preview", args)
if err != nil {
return microsub.Timeline{}, err
}

27
pkg/fetch/fetch.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
// Package fetch provides an API for fetching information about urls.
package fetch
@ -43,7 +25,7 @@ import (
)
// FeedHeader returns a new microsub.Feed with the information parsed from body.
func FeedHeader(fetcher Fetcher, fetchURL, contentType string, body io.Reader) (microsub.Feed, error) {
func FeedHeader(fetcher FetcherFunc, fetchURL, contentType string, body io.Reader) (microsub.Feed, error) {
log.Printf("ProcessContent %s\n", fetchURL)
log.Println("Found " + contentType)
@ -56,7 +38,7 @@ func FeedHeader(fetcher Fetcher, fetchURL, contentType string, body io.Reader) (
author, ok := jf2.SimplifyMicroformatDataAuthor(data)
if !ok {
if strings.HasPrefix(author.URL, "http") {
resp, err := fetcher.Fetch(author.URL)
resp, err := fetcher(author.URL)
if err != nil {
return feed, err
}
@ -66,9 +48,6 @@ func FeedHeader(fetcher Fetcher, fetchURL, contentType string, body io.Reader) (
md := microformats.Parse(resp.Body, u)
author, ok = jf2.SimplifyMicroformatDataAuthor(md)
if !ok {
log.Println("Could not simplify the author")
}
}
}
@ -129,7 +108,7 @@ func FeedHeader(fetcher Fetcher, fetchURL, contentType string, body io.Reader) (
}
// FeedItems returns the items from the url, parsed from body.
func FeedItems(fetcher Fetcher, fetchURL, contentType string, body io.Reader) ([]microsub.Item, error) {
func FeedItems(fetcher FetcherFunc, fetchURL, contentType string, body io.Reader) ([]microsub.Item, error) {
log.Printf("ProcessContent %s\n", fetchURL)
log.Println("Found " + contentType)

20
pkg/fetch/fetch_test.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package fetch
import (
@ -41,7 +23,7 @@ func TestFeedHeader(t *testing.T) {
</body>
</html>
`
feed, err := FeedHeader(FetcherFunc(fetcher), "https://example.com/", "text/html", strings.NewReader(doc))
feed, err := FeedHeader(fetcher, "https://example.com/", "text/html", strings.NewReader(doc))
if assert.NoError(t, err) {
assert.Equal(t, "feed", feed.Type)
assert.Equal(t, "Title", feed.Name)

28
pkg/fetch/fetcher.go
View File

@ -1,34 +1,6 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package fetch
import "net/http"
// Fetcher fetches urls
type Fetcher interface {
Fetch(url string) (*http.Response, error)
}
// FetcherFunc is a function that fetches an url
type FetcherFunc func(url string) (*http.Response, error)
// Fetch fetches an url and returns a response or error
func (ff FetcherFunc) Fetch(url string) (*http.Response, error) {
return ff(url)
}

18
pkg/indieauth/auth.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package indieauth
import (

18
pkg/jf2/reflect.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package jf2
import (

45
pkg/jf2/simplify.go
View File

@ -1,22 +1,21 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
// Package jf2 converts microformats to JF2
/*
ekster - microsub server
Copyright (C) 2018 Peter Stuifzand
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package jf2
import (
@ -89,14 +88,6 @@ func simplifyContent(k string, v []interface{}) *microsub.Content {
// CleanHTML removes white-space:pre from html
func CleanHTML(s string) (string, error) {
doc, err := html.Parse(strings.NewReader(s))
if err != nil {
return "", err
}
whitespaceRegex, err := regexp.Compile(`white-space:\s*pre`)
if err != nil {
return "", err
}
if err != nil {
return "", err
@ -110,7 +101,7 @@ func CleanHTML(s string) (string, error) {
if a.Key != "style" {
continue
}
if whitespaceRegex.MatchString(a.Val) {
if m, err := regexp.MatchString("white-space:\\s*pre", a.Val); err == nil && m {
removeIndex = i
break
}

30
pkg/jf2/simplify_test.go
View File

@ -1,19 +1,19 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
ekster - microsub server
Copyright (C) 2018 Peter Stuifzand
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package jf2_test

34
pkg/jsonfeed/jsonfeed.go
View File

@ -1,22 +1,18 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
// Package jsonfeed parses feeds in the jsonfeed format
// Package jsonfeed contains the types and a parse function for JSON feeds.
// Copyright (C) 2018 Peter Stuifzand
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package jsonfeed
import (

18
pkg/jsonfeed/jsonfeed_test.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package jsonfeed
import (

18
pkg/microsub/database.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package microsub
import (

27
pkg/microsub/protocol.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
// Package microsub describes the protocol methods of the Microsub protocol
package microsub
@ -100,15 +82,6 @@ type Item struct {
Refs map[string]Item `json:"refs,omitempty"`
ID string `json:"_id,omitempty"`
Read bool `json:"_is_read"`
Source *Source `json:"_source,omitempty"`
}
// Source is an Item source
type Source struct {
ID string `json:"_id"`
URL string `json:"url"`
Name string `json:"name"`
Photo string `json:"photo"`
}
// Pagination contains information about paging

18
pkg/microsub/protocol_test.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package microsub
import (

2
pkg/rss/ISO-8859-1.go
View File

@ -37,7 +37,7 @@ func (cs *charsetISO88591er) ReadByte() (b byte, err error) {
func (cs *charsetISO88591er) Read(p []byte) (int, error) {
// Use ReadByte method.
return 0, errors.New("use ReadByte()")
return 0, errors.New("Use ReadByte()")
}
func isCharsetISO88591(charset string) bool {

4
pkg/rss/rss.go
View File

@ -303,7 +303,7 @@ type Enclosure struct {
// Get uses http.Get to fetch an enclosure.
func (e *Enclosure) Get() (io.ReadCloser, error) {
if e == nil || e.URL == "" {
return nil, errors.New("no enclosure")
return nil, errors.New("No enclosure")
}
res, err := http.Get(e.URL)
@ -325,7 +325,7 @@ type Image struct {
// Get uses http.Get to fetch an image.
func (i *Image) Get() (io.ReadCloser, error) {
if i == nil || i.URL == "" {
return nil, errors.New("no image")
return nil, errors.New("No image")
}
res, err := http.Get(i.URL)

8
pkg/rss/rss_1.0.go
View File

@ -5,10 +5,8 @@ import (
"encoding/xml"
"fmt"
"sort"
"strings"
"time"
"golang.org/x/text/cases"
"golang.org/x/text/language"
)
func parseRSS1(data []byte) (*Feed, error) {
@ -31,8 +29,6 @@ func parseRSS1(data []byte) (*Feed, error) {
out.Description = channel.Description
out.Link = channel.Link
out.Image = channel.Image.Image()
titleCaser := cases.Title(language.English)
if channel.MinsToLive != 0 {
sort.Ints(channel.SkipHours)
next := time.Now().Add(time.Duration(channel.MinsToLive) * time.Minute)
@ -45,7 +41,7 @@ func parseRSS1(data []byte) (*Feed, error) {
for trying {
trying = false
for _, day := range channel.SkipDays {
if titleCaser.String(day) == next.Weekday().String() {
if strings.Title(day) == next.Weekday().String() {
next.Add(time.Duration(24-next.Hour()) * time.Hour)
trying = true
break

7
pkg/rss/rss_2.0.go
View File

@ -5,10 +5,8 @@ import (
"encoding/xml"
"fmt"
"sort"
"strings"
"time"
"golang.org/x/text/cases"
"golang.org/x/text/language"
)
func parseRSS2(data []byte) (*Feed, error) {
@ -40,7 +38,6 @@ func parseRSS2(data []byte) (*Feed, error) {
out.Image = channel.Image.Image()
if channel.MinsToLive != 0 {
titleCaser := cases.Title(language.English)
sort.Ints(channel.SkipHours)
next := time.Now().Add(time.Duration(channel.MinsToLive) * time.Minute)
for _, hour := range channel.SkipHours {
@ -52,7 +49,7 @@ func parseRSS2(data []byte) (*Feed, error) {
for trying {
trying = false
for _, day := range channel.SkipDays {
if titleCaser.String(day) == next.Weekday().String() {
if strings.Title(day) == next.Weekday().String() {
next.Add(time.Duration(24-next.Hour()) * time.Hour)
trying = true
break

5
pkg/rss/rss_2.0_test.go
View File

@ -1,6 +1,7 @@
package rss
import (
"fmt"
"io/ioutil"
"path/filepath"
"testing"
@ -76,7 +77,7 @@ func TestParseItemDateOK(t *testing.T) {
t.Fatalf("Parsing %s: %v", name, err)
}
if feed.Items[0].Date.String() != want {
if fmt.Sprintf("%s", feed.Items[0].Date) != want {
t.Errorf("%s: got %q, want %q", name, feed.Items[0].Date, want)
}
}
@ -99,7 +100,7 @@ func TestParseItemDateFailure(t *testing.T) {
t.Fatalf("Parsing %s: %v", name, err)
}
if feed.Items[1].Date.String() != want {
if fmt.Sprintf("%s", feed.Items[1].Date) != want {
t.Errorf("%s: got %q, want %q", name, feed.Items[1].Date, want)
}

71
pkg/server/microsub.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/*
Package server contains the microsub server itself. It implements http.Handler.
It follows the spec at https://indieweb.org/Microsub-spec.
@ -34,7 +16,7 @@ import (
)
var (
entryRegex = regexp.MustCompile(`^entry\[\d+\]$`)
entryRegex = regexp.MustCompile("^entry\\[\\d+\\]$")
)
// Constants used for the responses
@ -54,7 +36,7 @@ func respondJSON(w http.ResponseWriter, value interface{}) {
w.Header().Add("Content-Type", OutputContentType)
err := jw.Encode(value)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
http.Error(w, err.Error(), 500)
}
}
@ -87,8 +69,7 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if action == "channels" {
channels, err := h.backend.ChannelsGetList()
if err != nil {
log.Println(err)
http.Error(w, err.Error(), http.StatusInternalServerError)
http.Error(w, err.Error(), 500)
return
}
respondJSON(w, map[string][]microsub.Channel{
@ -97,16 +78,14 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} else if action == "timeline" {
timeline, err := h.backend.TimelineGet(values.Get("before"), values.Get("after"), values.Get("channel"))
if err != nil {
log.Println(err)
http.Error(w, err.Error(), http.StatusInternalServerError)
http.Error(w, err.Error(), 500)
return
}
respondJSON(w, timeline)
} else if action == "preview" {
timeline, err := h.backend.PreviewURL(values.Get("url"))
if err != nil {
log.Println(err)
http.Error(w, err.Error(), http.StatusInternalServerError)
http.Error(w, err.Error(), 500)
return
}
respondJSON(w, timeline)
@ -114,8 +93,7 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
channel := values.Get("channel")
following, err := h.backend.FollowGetList(channel)
if err != nil {
log.Println(err)
http.Error(w, err.Error(), http.StatusInternalServerError)
http.Error(w, err.Error(), 500)
return
}
respondJSON(w, map[string][]microsub.Feed{
@ -124,8 +102,7 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} else if action == "events" {
events, err := h.backend.Events()
if err != nil {
log.Println(err)
http.Error(w, "could not start sse connection", http.StatusInternalServerError)
http.Error(w, "could not start sse connection", 500)
}
// Remove this client from the map of connected clients
@ -135,7 +112,7 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}()
// Listen to connection close and un-register messageChan
notify := r.Context().Done()
notify := w.(http.CloseNotifier).CloseNotify()
go func() {
<-notify
@ -145,10 +122,10 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
err = sse.WriteMessages(w, events)
if err != nil {
log.Println(err)
http.Error(w, "internal server error", http.StatusInternalServerError)
http.Error(w, "internal server error", 500)
}
} else {
http.Error(w, fmt.Sprintf("unknown action %s", action), http.StatusBadRequest)
http.Error(w, fmt.Sprintf("unknown action %s\n", action), 400)
return
}
return
@ -164,8 +141,7 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if method == "delete" {
err := h.backend.ChannelsDelete(uid)
if err != nil {
log.Println(err)
http.Error(w, err.Error(), http.StatusInternalServerError)
http.Error(w, err.Error(), 500)
return
}
respondJSON(w, []string{})
@ -175,16 +151,14 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if uid == "" {
channel, err := h.backend.ChannelsCreate(name)
if err != nil {
log.Println(err)
http.Error(w, err.Error(), http.StatusInternalServerError)
http.Error(w, err.Error(), 500)
return
}
respondJSON(w, channel)
} else if name != "" {
channel, err := h.backend.ChannelsUpdate(uid, name)
if err != nil {
log.Println(err)
http.Error(w, err.Error(), http.StatusInternalServerError)
http.Error(w, err.Error(), 500)
return
}
respondJSON(w, channel)
@ -195,7 +169,7 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// h.HubIncomingBackend.CreateFeed(url, uid)
feed, err := h.backend.FollowURL(uid, url)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
http.Error(w, err.Error(), 500)
return
}
respondJSON(w, feed)
@ -204,17 +178,10 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
url := values.Get("url")
err := h.backend.UnfollowURL(uid, url)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
http.Error(w, err.Error(), 500)
return
}
respondJSON(w, []string{})
} else if action == "preview" {
timeline, err := h.backend.PreviewURL(values.Get("url"))
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
respondJSON(w, timeline)
} else if action == "search" {
query := values.Get("query")
channel := values.Get("channel")
@ -239,7 +206,6 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
})
return
}
log.Printf("Searching for %s in %s (%d results)", query, channel, len(items))
respondJSON(w, map[string]interface{}{
"query": query,
"items": items,
@ -269,21 +235,22 @@ func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if len(markAsRead) > 0 {
err := h.backend.MarkRead(channel, markAsRead)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
http.Error(w, err.Error(), 500)
return
}
} else {
log.Println("No uids specified for mark read")
}
} else {
http.Error(w, fmt.Sprintf("unknown method in timeline %s\n", method), http.StatusInternalServerError)
http.Error(w, fmt.Sprintf("unknown method in timeline %s\n", method), 500)
return
}
respondJSON(w, []string{})
} else {
http.Error(w, fmt.Sprintf("unknown action %s\n", action), http.StatusBadRequest)
http.Error(w, fmt.Sprintf("unknown action %s\n", action), 400)
}
return
}
return
}

18
pkg/server/microsub_test.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package server
import (

18
pkg/server/null.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package server
import (

18
pkg/sse/events.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package sse
import (

18
pkg/timeline/null.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package timeline
import "p83.nl/go/ekster/pkg/microsub"

134
pkg/timeline/postgres.go
View File

@ -1,31 +1,10 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package timeline
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"fmt"
"log"
"strconv"
"strings"
"time"
@ -54,10 +33,46 @@ func (p *postgresStream) Init() error {
return fmt.Errorf("database ping failed: %w", err)
}
row := conn.QueryRowContext(ctx, `SELECT "id" FROM "channels" WHERE "uid" = $1`, p.channel)
_, err = conn.ExecContext(ctx, `
CREATE TABLE IF NOT EXISTS "channels" (
"id" int primary key generated always as identity,
"name" varchar(255) unique,
"created_at" timestamp DEFAULT current_timestamp
);
`)
if err != nil {
return fmt.Errorf("create channels table failed: %w", err)
}
_, err = conn.ExecContext(ctx, `
CREATE TABLE IF NOT EXISTS "items" (
"id" int primary key generated always as identity,
"channel_id" int references "channels" on delete cascade,
"uid" varchar(512) not null unique,
"is_read" int default 0,
"data" json,
"created_at" timestamp DEFAULT current_timestamp,
"updated_at" timestamp,
"published_at" timestamp
);
`)
if err != nil {
return fmt.Errorf("create items table failed: %w", err)
}
_, err = conn.ExecContext(ctx, `INSERT INTO "channels" ("name", "created_at") VALUES ($1, DEFAULT)
ON CONFLICT DO NOTHING`, p.channel)
if err != nil {
return fmt.Errorf("create channel failed: %w", err)
}
row := conn.QueryRowContext(ctx, `SELECT "id" FROM "channels" WHERE "name" = $1`, p.channel)
if row == nil {
return fmt.Errorf("fetch channel failed: %w", err)
}
err = row.Scan(&p.channelID)
if err == sql.ErrNoRows {
return fmt.Errorf("channel %s not found: %w", p.channel, err)
if err != nil {
return fmt.Errorf("fetch channel failed while scanning: %w", err)
}
return nil
@ -86,16 +101,16 @@ WHERE "channel_id" = $1
log.Println(err)
} else {
args = append(args, b)
qb.WriteString(` AND "published_at" > $2`)
qb.WriteString(` AND "published_at" < $2`)
}
} else if after != "" {
b, err := time.Parse(time.RFC3339, after)
if err == nil {
args = append(args, b)
qb.WriteString(` AND "published_at" < $2`)
qb.WriteString(` AND "published_at" > $2`)
}
}
qb.WriteString(` ORDER BY "published_at" DESC LIMIT 20`)
qb.WriteString(` ORDER BY "published_at" DESC LIMIT 10`)
rows, err := conn.QueryContext(context.Background(), qb.String(), args...)
if err != nil {
@ -139,12 +154,9 @@ WHERE "channel_id" = $1
return tl, err
}
if len(tl.Items) > 0 && hasMoreBefore(conn, tl.Items[0].Published) {
tl.Paging.Before = tl.Items[0].Published
}
if hasMoreAfter(conn, last) {
tl.Paging.After = last
}
// TODO: should only be set of there are more items available
tl.Paging.Before = last
// tl.Paging.After = last
if tl.Items == nil {
tl.Items = []microsub.Item{}
@ -153,24 +165,6 @@ WHERE "channel_id" = $1
return tl, nil
}
func hasMoreBefore(conn *sql.Conn, before string) bool {
row := conn.QueryRowContext(context.Background(), `SELECT COUNT(*) FROM "items" WHERE "published_at" > $1`, before)
var count int
if err := row.Scan(&count); err == sql.ErrNoRows {
return false
}
return count > 0
}
func hasMoreAfter(conn *sql.Conn, after string) bool {
row := conn.QueryRowContext(context.Background(), `SELECT COUNT(*) FROM "items" WHERE "published_at" < $1`, after)
var count int
if err := row.Scan(&count); err == sql.ErrNoRows {
return false
}
return count > 0
}
// Count
func (p *postgresStream) Count() (int, error) {
ctx := context.Background()
@ -179,12 +173,16 @@ func (p *postgresStream) Count() (int, error) {
return -1, err
}
defer conn.Close()
var count int
row := conn.QueryRowContext(context.Background(), `SELECT COUNT(*) FROM items WHERE channel_id = $1 AND "is_read" = 0`, p.channelID)
err = row.Scan(&count)
if err != nil && err == sql.ErrNoRows {
if row == nil {
return 0, nil
}
var count int
err = row.Scan(&count)
if err != nil {
return -1, err
}
return count, nil
}
@ -205,34 +203,14 @@ func (p *postgresStream) AddItem(item microsub.Item) (bool, error) {
}
t = t2
}
if item.ID == "" {
// FIXME: This won't work when we receive the item multiple times
h := sha256.Sum256([]byte(fmt.Sprintf("%s:%d", p.channel, time.Now().UnixNano())))
item.UID = hex.EncodeToString(h[:])
}
var optFeedID sql.NullInt64
if item.Source == nil || item.Source.ID == "" {
optFeedID.Valid = false
optFeedID.Int64 = 0
} else {
feedID, err := strconv.ParseInt(item.Source.ID, 10, 64)
if err != nil {
optFeedID.Valid = false
optFeedID.Int64 = 0
} else {
optFeedID.Valid = true
optFeedID.Int64 = feedID
}
}
result, err := conn.ExecContext(context.Background(), `
INSERT INTO "items" ("channel_id", "feed_id", "uid", "data", "published_at", "created_at")
VALUES ($1, $2, $3, $4, $5, DEFAULT)
INSERT INTO "items" ("channel_id", "uid", "data", "published_at", "created_at")
VALUES ($1, $2, $3, $4, DEFAULT)
ON CONFLICT ON CONSTRAINT "items_uid_key" DO NOTHING
`, p.channelID, optFeedID, item.ID, &item, t)
`, p.channelID, item.ID, &item, t)
if err != nil {
return false, fmt.Errorf("insert item: %w", err)
return false, fmt.Errorf("while adding item: %w", err)
}
c, err := result.RowsAffected()
if err != nil {

24
pkg/timeline/redisset.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package timeline
import (
@ -132,12 +114,6 @@ func (timeline *redisSortedSetTimeline) AddItem(item microsub.Item) (bool, error
item.Published = time.Now().Format(time.RFC3339)
}
// Fix date when it almost matches with RFC3339, except the colon in the timezone
format := "2006-01-02T15:04:05Z0700"
if parsedDate, err := time.Parse(format, item.Published); err == nil {
item.Published = parsedDate.Format(time.RFC3339)
}
data, err := json.Marshal(item)
if err != nil {
return false, fmt.Errorf("couldn't marshal item for redis: %s", err)

18
pkg/timeline/redisstreams.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package timeline
import (

18
pkg/timeline/timeline.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
// Package timeline contains different types of timeline backends.
//
// "sorted-set" uses Redis sorted sets as a backend

22
pkg/util/algorithm.go
View File

@ -1,26 +1,7 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package util
import "reflect"
// Rotate rotates values of an array, between index f and l with midpoint k
func Rotate(a interface{}, f, k, l int) int {
swapper := reflect.Swapper(a)
if f == k {
@ -57,7 +38,6 @@ func Rotate(a interface{}, f, k, l int) int {
return ret
}
// StablePartition partitions elements of the array between indices f and l according to predicate p
func StablePartition(a interface{}, f, l int, p func(i int) bool) int {
n := l - f
@ -68,7 +48,7 @@ func StablePartition(a interface{}, f, l int, p func(i int) bool) int {
if n == 1 {
t := f
if p(f) {
t++
t += 1
}
return t
}

19
pkg/util/randkey.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package util
import (
@ -24,7 +6,6 @@ import (
const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
// RandStringBytes generates a random string of n characters
func RandStringBytes(n int) string {
b := make([]byte, n)
for i := range b {

18
pkg/websub/signature.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package websub
import (

18
pkg/websub/signature_test.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package websub
import (

18
pkg/websub/subscribe.go
View File

@ -1,21 +1,3 @@
/*
* Ekster is a microsub server
* Copyright (c) 2021 The Ekster authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package websub
import (

100
templates/auth.html Normal file
View File

@ -0,0 +1,100 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Ekster</title>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.1/css/bulma.min.css">
</head>
<body>
<section class="section">
<div class="container">
<nav class="navbar" role="navigation" aria-label="main navigation">
<div class="navbar-brand">
<a class="navbar-item" href="/">
Ekster
</a>
<a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false" data-target="menu">
<span aria-hidden="true"></span>
<span aria-hidden="true"></span>
<span aria-hidden="true"></span>
</a>
</div>
{{ if .Session.LoggedIn }}
<div id="menu" class="navbar-menu">
<a class="navbar-item" href="/settings">
Settings
</a>
<a class="navbar-item" href="/logs">
Logs
</a>
<a class="navbar-item" href="{{ .Session.Me }}">
Profile
</a>
</div>
{{ end }}
</nav>
<h1 class="title">Ekster - Microsub server</h1>
<div class="box">
<form action="/auth/approve" method="post">
<input type="hidden" name="state" value="{{ .State }}" />
<div class="field">
<label class="label">Client ID</label>
<div class="control">
<p>{{ .ClientID }}</p>
</div>
<div class="control">
<p>{{ .App.Name }}</p>
</div>
<div class="control">
<p><img src="{{ .App.IconURL }}" /></p>
</div>
</div>
<div class="field">
<label class="label">RedirectURI</label>
<div class="control">
<p>{{ .RedirectURI }}</p>
</div>
</div>
<div class="field">
<label class="label">Scope</label>
<div class="control">
<p>{{ .Scope }}</p>
</div>
</div>
<div class="field">
<label class="label">Select a channel</label>
<div class="control">
<div class="select">
<select name="channel">
{{ range .Channels }}
<option value="{{ .UID }}">{{ .Name }}</option>
{{ end }}
</select>
</div>
</div>
</div>
<div class="field">
<div class="control">
<button type="submit" name="accept" value="approve" class="button is-primary">
Approve
</button>
</div>
</div>
</form>
</div>
</div>
</section>
</body>
</html>

49
templates/base.html Normal file
View File

@ -0,0 +1,49 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>{{template "title" .}}</title>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.1/css/bulma.min.css">
</head>
<body>
<section class="section">
<div class="container">
<nav class="navbar" role="navigation" aria-label="main navigation">
<div class="navbar-brand">
<a class="navbar-item" href="/">
Ekster
</a>
<a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false" data-target="menu">
<span aria-hidden="true"></span>
<span aria-hidden="true"></span>
<span aria-hidden="true"></span>
</a>
</div>
{{ if .Session.LoggedIn }}
<div id="menu" class="navbar-menu">
<a class="navbar-item" href="/settings">
Settings
</a>
<a class="navbar-item" href="/logs">
Logs
</a>
<a class="navbar-item" href="{{ .Session.Me }}">
Profile
</a>
</div>
{{ end }}
</nav>
<h1 class="title">Ekster - Microsub server</h1>
{{template "content" .}}
</div>
</section>
</body>
</html>
{{define "header"}}{{end}}
{{define "content"}}{{end}}
{{define "footer"}}{{end}}

122
templates/channel.html Normal file
View File

@ -0,0 +1,122 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Ekster</title>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.1/css/bulma.min.css">
</head>
<body>
<section class="section">
<div class="container">
<nav class="navbar" role="navigation" aria-label="main navigation">
<div class="navbar-brand">
<a class="navbar-item" href="/">
Ekster
</a>
<a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false" data-target="menu">
<span aria-hidden="true"></span>
<span aria-hidden="true"></span>
<span aria-hidden="true"></span>
</a>
</div>
{{ if .Session.LoggedIn }}
<div id="menu" class="navbar-menu">
<a class="navbar-item" href="/settings">
Settings
</a>
<a class="navbar-item" href="/logs">
Logs
</a>
<a class="navbar-item" href="{{ .Session.Me }}">
Profile
</a>
</div>
{{ end }}
</nav>
<h1 class="title">Ekster - Microsub server</h1>
{{ $channel := .CurrentChannel }}
<nav class="breadcrumb" aria-label="breadcrumbs">
<ul>
<li><a href="/settings">Settings</a></li>
<li class="is-active"><a href="/setttings/channel?uid={{ .CurrentChannel }}">{{ $channel.Name }}</a></li>
</ul>
</nav>
<h2 class="subtitle is-2">{{ $channel.Name }}</h2>
<div class="columns">
<div class="column">
<h3 class="title is-4">Settings</h3>
<form action="/settings/channel" method="post">
<input type="hidden" name="uid" value="{{ .CurrentChannel.UID }}" />
<div class="field">
<label class="label" for="exclude_regex">Blocking Regex</label>
<div class="control">
<input type="text" class="input" id="exclude_regex" name="exclude_regex" value="{{ .CurrentSetting.ExcludeRegex }}" placeholder="enter regex to block" />
</div>
</div>
<div class="field">
<label class="label" for="include_regex">Tracking Regex</label>
<div class="control">
<input type="text" class="input" id="include_regex" name="include_regex" value="{{ .CurrentSetting.IncludeRegex }}" placeholder="enter regex to track items" />
</div>
</div>
<div class="field">
<label class="label" for="type">Channel Type</label>
<div class="control">
<div class="select">
<select name="type" id="type">
<option value="null" {{if eq (.CurrentSetting.ChannelType) "null" }}selected{{end}}>Null</option>
<option value="sorted-set" {{if eq (.CurrentSetting.ChannelType) "sorted-set" }}selected{{end}}>Sorted Set</option>
<option value="stream" {{if eq (.CurrentSetting.ChannelType) "stream" }}selected{{end}}>Streams</option>
<option value="postgres-stream" {{if eq (.CurrentSetting.ChannelType) "postgres-stream" }}selected{{end}}>Postgres Stream</option>
</select>
</div>
</div>
</div>
<div class="field">
<label for="exclude_type" class="label">Exclude Types</label>
<div class="control">
<div class="select is-multiple">
<select name="exclude_type" id="exclude_type" multiple>
{{ range $key, $excluded := $.ExcludedTypes }}
<option value="{{ $key }}" {{ if $excluded }}selected="selected"{{ end }}>{{ index $.ExcludedTypeNames $key }}</option>
{{ end }}
</select>
</div>
</div>
</div>
<div class="field">
<div class="control">
<button type="submit" class="button is-primary">Save</button>
</div>
</div>
</form>
</div>
<div class="column">
<h3 class="title is-4">Feeds</h3>
<div class="channel">
{{ range .Feeds }}
<div class="feed box">
<div class="name">
<a href="{{ .URL }}">{{ .URL }}</a>
</div>
</div>
{{ else }}
<div class="no-channels">No feeds</div>
{{ end }}
</div>
</div>
</div>
</div>
</section>
</body>
</html>

71
templates/index.html Normal file
View File

@ -0,0 +1,71 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Ekster</title>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.1/css/bulma.min.css">
<link rel="micropub" href="{{ .Baseurl }}/micropub" />
<link rel="authorization_endpoint" href="{{ .Baseurl }}/auth" />
<link rel="token_endpoint" href="{{ .Baseurl }}/auth/token" />
</head>
<body>
<section class="section">
<div class="container">
<nav class="navbar" role="navigation" aria-label="main navigation">
<div class="navbar-brand">
<a class="navbar-item" href="/">
Ekster
</a>
<a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false" data-target="menu">
<span aria-hidden="true"></span>
<span aria-hidden="true"></span>
<span aria-hidden="true"></span>
</a>
</div>
{{ if .Session.LoggedIn }}
<div id="menu" class="navbar-menu">
<a class="navbar-item" href="/settings">
Settings
</a>
<a class="navbar-item" href="/logs">
Logs
</a>
<a class="navbar-item" href="{{ .Session.Me }}">
Profile
</a>
</div>
{{ end }}
</nav>
<h1 class="title">Ekster - Microsub server</h1>
{{ if .Session.LoggedIn }}
<h2 class="title">Logout</h2>
<form action="/session/logout" method="post">
<button type="submit" class="button is-info">Logout</button>
</form>
{{ else }}
<h2 class="title">Sign in to Ekster</h2>
<form action="/session" method="post">
<div class="field">
<label class="label" for="url"></label>
<div class="control">
<input type="text" name="url" id="url" class="input" placeholder="https://example.com/">
</div>
</div>
<div class="field is-grouped">
<div class="control">
<button type="submit" class="button is-info">Login</button>
</div>
</div>
</form>
{{ end }}
</div>
</section>
</body>
</html>

50
templates/logs.html Normal file
View File

@ -0,0 +1,50 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Ekster</title>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.1/css/bulma.min.css">
</head>
<body>
<section class="section">
<div class="container">
<nav class="navbar" role="navigation" aria-label="main navigation">
<div class="navbar-brand">
<a class="navbar-item" href="/">
Ekster
</a>
<a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false" data-target="menu">
<span aria-hidden="true"></span>
<span aria-hidden="true"></span>
<span aria-hidden="true"></span>
</a>
</div>
{{ if .Session.LoggedIn }}
<div id="menu" class="navbar-menu">
<a class="navbar-item" href="/settings">
Settings
</a>
<a class="navbar-item" href="/logs">
Logs
</a>
<a class="navbar-item" href="{{ .Session.Me }}">
Profile
</a>
</div>
{{ end }}
</nav>
<h1 class="title">Ekster - Microsub server</h1>
<h2 class="subtitle">Logs</h2>
<p>Logs</p>
</div>
</section>
</body>
</html>

63
templates/settings.html Normal file
View File

@ -0,0 +1,63 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Ekster</title>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.1/css/bulma.min.css">
</head>
<body>
<section class="section">
<div class="container">
<nav class="navbar" role="navigation" aria-label="main navigation">
<div class="navbar-brand">
<a class="navbar-item" href="/">
Ekster
</a>
<a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false" data-target="menu">
<span aria-hidden="true"></span>
<span aria-hidden="true"></span>
<span aria-hidden="true"></span>
</a>
</div>
{{ if .Session.LoggedIn }}
<div id="menu" class="navbar-menu">
<a class="navbar-item" href="/settings">
Settings
</a>
<a class="navbar-item" href="/logs">
Logs
</a>
<a class="navbar-item" href="{{ .Session.Me }}">
Profile
</a>
</div>
{{ end }}
</nav>
<h1 class="title">Ekster - Microsub server</h1>
{{ if .Session.LoggedIn }}
{{ end }}
<h2 class="subtitle">Channels</h2>
<div class="channels">
{{ range .Channels }}
<div class="channel box">
<div class="name">
<a href="/settings/channel?uid={{ .UID }}">
{{ .Name }}
</a>
</div>
</div>
{{ else }}
<div class="no-channels">No channels</div>
{{ end }}
</div>
</div>
</section>
</body>
</html>