Add stream type postgres-stream

.gitignore

@@ -7,3 +7,6 @@ cmd/ek/ek
 /eksterd
 
 backend.json
+
+data/
+database-data/

README.md

@@ -10,7 +10,6 @@ There are two methods for installing and running ekster.
 
 Download the binaries from the [latest release](https://github.cGom/pstuifzand/ekster/releases/) on Github.
 
-
 ### Method 2: Install ekster from source with Go
 
 ekster is build using [go](https://golang.org). To be able to install ekster
@@ -45,10 +44,10 @@ You can now access `eksterd` on port `8090`. To really use it, you should proxy
 
 ### Method 3: Using Docker / Docker Compose
 
-It's now also possible to use docker-compose to start an ekster server. Create an empty directory. 
+It's now also possible to use docker-compose to start an ekster server. Create an empty directory.
-Download [docker-compose.yml](https://raw.githubusercontent.com/pstuifzand/ekster/master/docker-compose.yml) from Github 
+Download [docker-compose.yml](https://raw.githubusercontent.com/pstuifzand/ekster/master/docker-compose.yml) from Github
 and run the following commands in that directory.
-    
+
     docker-compose pull
     docker-compose run web new
     # edit the backend.json file according to the instructions
@@ -59,7 +58,7 @@ After editing, you can run `docker-compose up` to start the server. This will st
 so that you can run the program without problems. By default it will choose a random port, to run the server.
 To make it really useful, you need to run this on an internet connected server and choose a fixed port.
 
-The nicest way to run this docker-compose environment is with a proxy in the front. You can run ekster behind 
+The nicest way to run this docker-compose environment is with a proxy in the front. You can run ekster behind
 [nginx-proxy](https://github.com/jwilder/nginx-proxy).
 
 ## When ekster is running
@@ -74,7 +73,7 @@ you use to proxy the server.
 The microsub server responds to the `/microsub` url with the micropub protocol.
 You can use `ek` to talk to the endpoint.
 
-It's also possible to visit the microsub server with your browser, there are a few ways to 
+It's also possible to visit the microsub server with your browser, there are a few ways to
 change settings.
 
 ## Commands

docker-compose.yml

@@ -3,6 +3,16 @@ services:
   redis:
     image: "redis:5"
 
+  database:
+    image: postgres
+    volumes:
+    - ./database-data:/var/lib/postgresql/data
+    environment:
+      POSTGRES_DB: ekster
+      POSTGRES_USER: ekster
+      POSTGRES_PASSWORD: simple
+      POSTGRES_HOST_AUTH_METHOD: trust
+
   web:
     image: "pstuifzand/ekster:alpine"
     working_dir: /opt/microsub
@@ -10,6 +20,8 @@ services:
       - redis:redis
     volumes:
       - ./data:/opt/microsub
+      - ./templates:/app/templates
+      - ./eksterd:/app/eksterd
     entrypoint: /app/eksterd
     command: -auth=false -port 80 -templates /app/templates
     ports:
@@ -17,5 +29,5 @@ services:
     environment:
       - "FEEDBIN_USER="
       - "FEEDBIN_PASS="
-      - "EKSTER_BASEURL="
+      - "EKSTER_BASEURL=http://localhost:8089/"
       - "EKSTER_TEMPLATES=/app/templates"

go.mod

@@ -7,9 +7,9 @@ require (
 	github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394
 	github.com/gilliek/go-opml v1.0.0
 	github.com/gomodule/redigo v1.8.2
+	github.com/lib/pq v1.10.1
 	github.com/pkg/errors v0.9.1
 	github.com/stretchr/testify v1.5.1
-	github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect
 	golang.org/x/net v0.0.0-20200707034311-ab3426394381
 	willnorris.com/go/microformats v1.1.0
 )

go.sum

@@ -10,11 +10,12 @@ github.com/gilliek/go-opml v1.0.0 h1:X8xVjtySRXU/x6KvaiXkn7OV3a4DHqxY8Rpv6U/JvCY
 github.com/gilliek/go-opml v1.0.0/go.mod h1:fOxmtlzyBvUjU6bjpdjyxCGlWz+pgtAHrHf/xRZl3lk=
 github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k=
 github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0=
-github.com/gomodule/redigo/redis v0.0.0-do-not-use h1:J7XIp6Kau0WoyT4JtXHT3Ei0gA1KkSc6bc87j9v9WIo=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/lib/pq v1.10.1 h1:6VXZrLU0jHBYyAqrSPa+MgPfnSvTPuMgK+k0o5kVFWo=
+github.com/lib/pq v1.10.1/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -22,8 +23,6 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y=
-github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

pkg/fetch/fetch.go

@@ -37,12 +37,12 @@ func FeedHeader(fetcher FetcherFunc, fetchURL, contentType string, body io.Reade
 		author, ok := jf2.SimplifyMicroformatDataAuthor(data)
 		if !ok {
 			if strings.HasPrefix(author.URL, "http") {
-				resp, err := fetcher(fetchURL)
+				resp, err := fetcher(author.URL)
 				if err != nil {
 					return feed, err
 				}
 				defer resp.Body.Close()
-				u, _ := url.Parse(fetchURL)
+				u, _ := url.Parse(author.URL)
 
 				md := microformats.Parse(resp.Body, u)
 

pkg/timeline/timeline.go

@@ -7,6 +7,7 @@ package timeline
 
 import (
 	"encoding/json"
+	"log"
 
 	"p83.nl/go/ekster/pkg/microsub"
 
@@ -26,7 +27,7 @@ type Backend interface {
 	// MarkUnread(uids []string) error
 }
 
-// Create creates a channel of the specfied type. Return nil when the type
+// Create creates a channel of the specified type. Return nil when the type
 // is not known.
 func Create(channel, timelineType string, pool *redis.Pool) Backend {
 	if timelineType == "sorted-set" {
@@ -56,6 +57,16 @@ func Create(channel, timelineType string, pool *redis.Pool) Backend {
 		return timeline
 	}
 
+	if timelineType == "postgres-stream" {
+		timeline := &PostgresStream{channel: channel}
+		err := timeline.Init()
+		if err != nil {
+			log.Printf("Error while creating %s: %v", channel, err)
+			return nil
+		}
+		return timeline
+	}
+
 	return nil
 }
 

templates/channel.html

@@ -75,6 +75,7 @@
                                         <option value="null" {{if eq (.CurrentSetting.ChannelType) "null" }}selected{{end}}>Null</option>
                                         <option value="sorted-set" {{if eq (.CurrentSetting.ChannelType) "sorted-set" }}selected{{end}}>Sorted Set</option>
                                         <option value="stream" {{if eq (.CurrentSetting.ChannelType) "stream" }}selected{{end}}>Streams</option>
+                                        <option value="postgres-stream" {{if eq (.CurrentSetting.ChannelType) "postgres-stream" }}selected{{end}}>Postgres Stream</option>
                                     </select>
                                 </div>
                             </div>
@@ -84,7 +85,7 @@
                             <div class="control">
                                 <div class="select is-multiple">
                                     <select name="exclude_type" id="exclude_type" multiple>
-                                        {{ range $key, $excluded := .ExcludedTypes }}
+                                        {{ range $key, $excluded := $.ExcludedTypes }}
                                             <option value="{{ $key }}" {{ if $excluded }}selected="selected"{{ end }}>{{ index $.ExcludedTypeNames $key }}</option>
                                         {{ end }}
                                     </select>