Remove "reader" command

README.md

@@ -6,7 +6,12 @@ a [Microsub](https://indieweb.org/Microsub) server
 
 There are two methods for installing and running ekster.
 
-### Method 1: Install ekster (from source)
+### Method 1: From binaries
+
+Download the binaries from the [latest release](https://github.com/pstuifzand/ekster/releases/) on Github.
+
+
+### Method 2: Install ekster from source with Go
 
 ekster is build using [go](https://golang.org). To be able to install ekster
 you need a Go environment. Use these commands to install the programs.
@@ -38,7 +43,7 @@ Start eksterd and pass the redis and port arguments.
 You can now access `eksterd` on port `8090`. To really use it, you should proxy
 `eksterd` behind a HTTP reverse proxy on port 80, or 443.
 
-### Method 2: Using Docker / Docker Compose
+### Method 3: Using Docker / Docker Compose
 
 It's now also possible to use docker-compose to start an ekster server. Create an empty directory. 
 Download [docker-compose.yml](https://raw.githubusercontent.com/pstuifzand/ekster/master/docker-compose.yml) from Github 

cmd/ek/main.go

@@ -314,8 +314,8 @@ func performCommands(sub microsub.Microsub, commands []string) {
 	}
 
 	if len(commands) == 2 && commands[0] == "preview" {
-		url := commands[1]
-		timeline, err := sub.PreviewURL(url)
+		u := commands[1]
+		timeline, err := sub.PreviewURL(u)
 
 		if err != nil {
 			log.Fatalf("An error occurred: %s\n", err)
@@ -338,8 +338,8 @@ func performCommands(sub microsub.Microsub, commands []string) {
 
 	if len(commands) == 3 && commands[0] == "follow" {
 		uid := commands[1]
-		url := commands[2]
-		_, err := sub.FollowURL(uid, url)
+		u := commands[2]
+		_, err := sub.FollowURL(uid, u)
 		if err != nil {
 			log.Fatalf("An error occurred: %s\n", err)
 		}
@@ -348,8 +348,8 @@ func performCommands(sub microsub.Microsub, commands []string) {
 
 	if len(commands) == 3 && commands[0] == "unfollow" {
 		uid := commands[1]
-		url := commands[2]
-		err := sub.UnfollowURL(uid, url)
+		u := commands[2]
+		err := sub.UnfollowURL(uid, u)
 		if err != nil {
 			log.Fatalf("An error occurred: %s\n", err)
 		}
@@ -359,72 +359,9 @@ func performCommands(sub microsub.Microsub, commands []string) {
 		filetype := commands[1]
 
 		if filetype == "opml" {
-			output := opml.OPML{}
-			output.Head.Title = "Microsub channels and feeds"
-			output.Head.DateCreated = time.Now().Format(time.RFC3339)
-			output.Version = "1.0"
-
-			channels, err := sub.ChannelsGetList()
-			if err != nil {
-				log.Fatalf("An error occurred: %s\n", err)
-			}
-
-			for _, c := range channels {
-				var feeds []opml.Outline
-				list, err := sub.FollowGetList(c.UID)
-				if err != nil {
-					log.Fatalf("An error occurred: %s\n", err)
-				}
-				for _, f := range list {
-					feeds = append(feeds, opml.Outline{
-						Title:   f.Name,
-						Text:    f.Name,
-						Type:    f.Type,
-						URL:     f.URL,
-						HTMLURL: f.URL,
-						XMLURL:  f.URL,
-					})
-				}
-
-				output.Body.Outlines = append(output.Body.Outlines, opml.Outline{
-					Text:     c.Name,
-					Title:    c.Name,
-					Outlines: feeds,
-				})
-			}
-
-			xml, err := output.XML()
-			if err != nil {
-				log.Fatalf("An error occurred: %s\n", err)
-			}
-			os.Stdout.WriteString(xml)
+			exportOpmlFromMicrosub(sub)
 		} else if filetype == "json" {
-			contents := Export{Version: "1.0", Generator: "ek version " + Version}
-
-			channels, err := sub.ChannelsGetList()
-			if err != nil {
-				log.Fatalf("An error occurred: %s\n", err)
-			}
-
-			for _, c := range channels {
-				contents.Channels = append(contents.Channels, ExportChannel{UID: c.UID, Name: c.Name})
-			}
-
-			contents.Feeds = make(map[string][]ExportFeed)
-
-			for _, c := range channels {
-				list, err := sub.FollowGetList(c.UID)
-				if err != nil {
-					log.Fatalf("An error occurred: %s\n", err)
-				}
-				for _, f := range list {
-					contents.Feeds[c.UID] = append(contents.Feeds[c.UID], ExportFeed(f.URL))
-				}
-			}
-			err = json.NewEncoder(os.Stdout).Encode(&contents)
-			if err != nil {
-				log.Fatalf("An error occurred: %s\n", err)
-			}
+			exportJsonFromMicrosub(sub)
 		} else {
 			log.Fatalf("unsupported filetype %q", filetype)
 		}
@@ -435,138 +372,9 @@ func performCommands(sub microsub.Microsub, commands []string) {
 		filename := commands[2]
 
 		if filetype == "opml" {
-			channelMap := make(map[string]microsub.Channel)
-
-			channels, err := sub.ChannelsGetList()
-			if err != nil {
-				log.Fatalf("an error occurred: %s\n", err)
-			}
-
-			for _, c := range channels {
-				channelMap[c.Name] = c
-			}
-
-			xml, err := opml.NewOPMLFromFile(filename)
-			if err != nil {
-				log.Fatalf("An error occurred: %s\n", err)
-			}
-
-			for _, c := range xml.Body.Outlines {
-				if c.HTMLURL != "" {
-					log.Printf("First row item has url: %s\n", c.HTMLURL)
-					continue
-				}
-				if len(c.Outlines) == 0 {
-					continue
-				}
-
-				uid := ""
-
-				if ch, e := channelMap[c.Text]; !e {
-					channelCreated, err := sub.ChannelsCreate(c.Text)
-					if err != nil {
-						log.Printf("An error occurred: %q\n", err)
-						continue
-					}
-
-					uid = channelCreated.UID
-					log.Printf("Channel created: %s\n", c.Text)
-				} else {
-					uid = ch.UID
-				}
-
-				feedMap := make(map[string]bool)
-
-				feeds, err := sub.FollowGetList(uid)
-				if err != nil {
-					log.Fatalf("An error occurred: %q\n", err)
-				}
-
-				for _, f := range feeds {
-					feedMap[f.URL] = true
-				}
-
-				for _, f := range c.Outlines {
-					if f.HTMLURL == "" {
-						log.Println("Missing url on second row item")
-						continue
-					}
-
-					if _, e := feedMap[f.HTMLURL]; !e {
-						_, err := sub.FollowURL(uid, f.HTMLURL)
-						if err != nil {
-							log.Printf("An error occurred: %q\n", err)
-							continue
-						}
-						log.Printf("Feed followed: %s\n", f.HTMLURL)
-					}
-				}
-			}
+			importOpmlIntoMicrosub(sub, filename)
 		} else if filetype == "json" {
-			var export Export
-
-			f, err := os.Open(filename)
-			if err != nil {
-				log.Fatalf("can't open file %s: %s", filename, err)
-			}
-			defer f.Close()
-
-			err = json.NewDecoder(f).Decode(&export)
-			if err != nil {
-				log.Fatalf("error while reading %s: %s", filename, err)
-			}
-
-			channelMap := make(map[string]microsub.Channel)
-
-			channels, err := sub.ChannelsGetList()
-			if err != nil {
-				log.Fatalf("an error occurred: %s\n", err)
-			}
-
-			for _, c := range channels {
-				channelMap[c.Name] = c
-			}
-
-			for _, c := range export.Channels {
-				uid := ""
-
-				if ch, e := channelMap[c.Name]; !e {
-					channelCreated, err := sub.ChannelsCreate(c.Name)
-					if err != nil {
-						log.Printf("An error occurred: %q\n", err)
-						continue
-					}
-
-					uid = channelCreated.UID
-					log.Printf("Channel created: %s\n", c.Name)
-				} else {
-					uid = ch.UID
-				}
-
-				feedMap := make(map[string]bool)
-
-				feeds, err := sub.FollowGetList(uid)
-				if err != nil {
-					log.Fatalf("An error occurred: %q\n", err)
-				}
-
-				for _, f := range feeds {
-					feedMap[f.URL] = true
-				}
-
-				for _, feed := range export.Feeds[uid] {
-
-					if _, e := feedMap[string(feed)]; !e {
-						_, err := sub.FollowURL(uid, string(feed))
-						if err != nil {
-							log.Printf("An error occurred: %q\n", err)
-							continue
-						}
-						log.Printf("Feed followed: %s\n", string(feed))
-					}
-				}
-			}
-
+			importJsonIntoMicrosub(sub, filename)
 		} else {
 			log.Fatalf("unsupported filetype %q", filetype)
 		}
@@ -577,6 +385,196 @@ func performCommands(sub microsub.Microsub, commands []string) {
 	}
 }
 
+func exportOpmlFromMicrosub(sub microsub.Microsub) {
+	output := opml.OPML{}
+	output.Head.Title = "Microsub channels and feeds"
+	output.Head.DateCreated = time.Now().Format(time.RFC3339)
+	output.Version = "1.0"
+	channels, err := sub.ChannelsGetList()
+	if err != nil {
+		log.Fatalf("An error occurred: %s\n", err)
+	}
+	for _, c := range channels {
+		var feeds []opml.Outline
+		list, err := sub.FollowGetList(c.UID)
+		if err != nil {
+			log.Fatalf("An error occurred: %s\n", err)
+		}
+		for _, f := range list {
+			feeds = append(feeds, opml.Outline{
+				Title:   f.Name,
+				Text:    f.Name,
+				Type:    f.Type,
+				URL:     f.URL,
+				HTMLURL: f.URL,
+				XMLURL:  f.URL,
+			})
+		}
+
+		output.Body.Outlines = append(output.Body.Outlines, opml.Outline{
+			Text:     c.Name,
+			Title:    c.Name,
+			Outlines: feeds,
+		})
+	}
+	xml, err := output.XML()
+	if err != nil {
+		log.Fatalf("An error occurred: %s\n", err)
+	}
+	os.Stdout.WriteString(xml)
+}
+
+func exportJsonFromMicrosub(sub microsub.Microsub) {
+	contents := Export{Version: "1.0", Generator: "ek version " + Version}
+	channels, err := sub.ChannelsGetList()
+	if err != nil {
+		log.Fatalf("An error occurred: %s\n", err)
+	}
+	for _, c := range channels {
+		contents.Channels = append(contents.Channels, ExportChannel{UID: c.UID, Name: c.Name})
+	}
+	contents.Feeds = make(map[string][]ExportFeed)
+	for _, c := range channels {
+		list, err := sub.FollowGetList(c.UID)
+		if err != nil {
+			log.Fatalf("An error occurred: %s\n", err)
+		}
+		for _, f := range list {
+			contents.Feeds[c.UID] = append(contents.Feeds[c.UID], ExportFeed(f.URL))
+		}
+	}
+	err = json.NewEncoder(os.Stdout).Encode(&contents)
+	if err != nil {
+		log.Fatalf("An error occurred: %s\n", err)
+	}
+}
+
+func importJsonIntoMicrosub(sub microsub.Microsub, filename string) {
+	var export Export
+	f, err := os.Open(filename)
+	if err != nil {
+		log.Fatalf("can't open file %s: %s", filename, err)
+	}
+	defer f.Close()
+	err = json.NewDecoder(f).Decode(&export)
+	if err != nil {
+		log.Fatalf("error while reading %s: %s", filename, err)
+	}
+	channelMap := make(map[string]microsub.Channel)
+	channels, err := sub.ChannelsGetList()
+	if err != nil {
+		log.Fatalf("an error occurred: %s\n", err)
+	}
+	for _, c := range channels {
+		channelMap[c.Name] = c
+	}
+	for _, c := range export.Channels {
+		uid := ""
+
+		if ch, e := channelMap[c.Name]; !e {
+			channelCreated, err := sub.ChannelsCreate(c.Name)
+			if err != nil {
+				log.Printf("An error occurred: %q\n", err)
+				continue
+			}
+
+			uid = channelCreated.UID
+			log.Printf("Channel created: %s\n", c.Name)
+		} else {
+			uid = ch.UID
+		}
+
+		feedMap := make(map[string]bool)
+
+		feeds, err := sub.FollowGetList(uid)
+		if err != nil {
+			log.Fatalf("An error occurred: %q\n", err)
+		}
+
+		for _, f := range feeds {
+			feedMap[f.URL] = true
+		}
+
+		for _, feed := range export.Feeds[uid] {
+
+			if _, e := feedMap[string(feed)]; !e {
+				_, err := sub.FollowURL(uid, string(feed))
+				if err != nil {
+					log.Printf("An error occurred: %q\n", err)
+					continue
+				}
+				log.Printf("Feed followed: %s\n", string(feed))
+			}
+		}
+	}
+}
+
+func importOpmlIntoMicrosub(sub microsub.Microsub, filename string) {
+	channelMap := make(map[string]microsub.Channel)
+	channels, err := sub.ChannelsGetList()
+	if err != nil {
+		log.Fatalf("an error occurred: %s\n", err)
+	}
+	for _, c := range channels {
+		channelMap[c.Name] = c
+	}
+	xml, err := opml.NewOPMLFromFile(filename)
+	if err != nil {
+		log.Fatalf("An error occurred: %s\n", err)
+	}
+	for _, c := range xml.Body.Outlines {
+		if c.HTMLURL != "" {
+			log.Printf("First row item has url: %s\n", c.HTMLURL)
+			continue
+		}
+		if len(c.Outlines) == 0 {
+			continue
+		}
+
+		uid := ""
+
+		if ch, e := channelMap[c.Text]; !e {
+			channelCreated, err := sub.ChannelsCreate(c.Text)
+			if err != nil {
+				log.Printf("An error occurred: %q\n", err)
+				continue
+			}
+
+			uid = channelCreated.UID
+			log.Printf("Channel created: %s\n", c.Text)
+		} else {
+			uid = ch.UID
+		}
+
+		feedMap := make(map[string]bool)
+
+		feeds, err := sub.FollowGetList(uid)
+		if err != nil {
+			log.Fatalf("An error occurred: %q\n", err)
+		}
+
+		for _, f := range feeds {
+			feedMap[f.URL] = true
+		}
+
+		for _, f := range c.Outlines {
+			if f.HTMLURL == "" {
+				log.Println("Missing url on second row item")
+				continue
+			}
+
+			if _, e := feedMap[f.HTMLURL]; !e {
+				_, err := sub.FollowURL(uid, f.HTMLURL)
+				if err != nil {
+					log.Printf("An error occurred: %q\n", err)
+					continue
+				}
+				log.Printf("Feed followed: %s\n", f.HTMLURL)
+			}
+		}
+	}
+}
+
 func showItem(item *microsub.Item) {
 	if item.Name != "" {
 		fmt.Printf("%s - ", item.Name)

cmd/eksterd/main.go

@@ -114,6 +114,11 @@ func main() {
 
 	if createBackend {
 		backend = createMemoryBackend()
+
+		log.Println(`Config file "backend.json" is created in the current directory.`)
+		log.Println(`Update "Me" variable to your website address "https://example.com/"`)
+		log.Println(`Update "TokenEndpoint" variable to the address of your token endpoint "https://example.com/token"`)
+
 		return
 	}
 

cmd/eksterd/memory.go

@@ -41,6 +41,8 @@ import (
 	"willnorris.com/go/microformats"
 )
 
+const DefaultPrio = 9999999
+
 type memoryBackend struct {
 	hubIncomingBackend
 
@@ -108,23 +110,24 @@ func (b *memoryBackend) load() error {
 		return err
 	}
 
+	return nil
+}
+
+func (b *memoryBackend) refreshChannels() {
 	conn := pool.Get()
 	defer conn.Close()
 
-	conn.Do("SETNX", "channel_sortorder_notifications", 1)
-
 	conn.Do("DEL", "channels")
 
-	b.lock.RLock()
-	defer b.lock.RUnlock()
+	updateChannelInRedis(conn, "notifications", 1)
 
+	b.lock.RLock()
 	for uid, channel := range b.Channels {
 		log.Printf("loading channel %s - %s\n", uid, channel.Name)
-		conn.Do("SADD", "channels", uid)
-		conn.Do("SETNX", "channel_sortorder_"+uid, 99999)
+		updateChannelInRedis(conn, channel.UID, DefaultPrio)
 	}
 
-	return nil
+	b.lock.RUnlock()
 }
 
 func (b *memoryBackend) save() {
@@ -145,6 +148,7 @@ func loadMemoryBackend() microsub.Microsub {
 		log.Printf("Error while loadingbackend: %v\n", err)
 		return nil
 	}
+	backend.refreshChannels()
 
 	return backend
 }
@@ -153,25 +157,23 @@ func createMemoryBackend() microsub.Microsub {
 	backend := memoryBackend{}
 	backend.lock.Lock()
 
-	backend.Channels = make(map[string]microsub.Channel)
 	backend.Feeds = make(map[string][]microsub.Feed)
 	channels := []microsub.Channel{
 		{UID: "notifications", Name: "Notifications"},
 		{UID: "home", Name: "Home"},
 	}
+
+	backend.Channels = make(map[string]microsub.Channel)
 	for _, c := range channels {
 		backend.Channels[c.UID] = c
 	}
+
 	backend.NextUid = 1000000
 	backend.Me = "https://example.com/"
 
 	backend.lock.Unlock()
 
 	backend.save()
-
-	log.Println(`Config file "backend.json" is created in the current directory.`)
-	log.Println(`Update "Me" variable to your website address "https://example.com/"`)
-	log.Println(`Update "TokenEndpoint" variable to the address of your token endpoint "https://example.com/token"`)
 	return &backend
 }
 
@@ -208,23 +210,13 @@ func (b *memoryBackend) ChannelsGetList() ([]microsub.Channel, error) {
 func (b *memoryBackend) ChannelsCreate(name string) (microsub.Channel, error) {
 	defer b.save()
 
+	channel := b.createChannel(name)
+	b.setChannel(channel)
+
 	conn := pool.Get()
 	defer conn.Close()
 
-	uid := fmt.Sprintf("%04d", b.NextUid)
-	channel := microsub.Channel{
-		UID:  uid,
-		Name: name,
-	}
-
-	b.lock.Lock()
-	b.Channels[channel.UID] = channel
-	b.Feeds[channel.UID] = []microsub.Feed{}
-	b.NextUid++
-	b.lock.Unlock()
-
-	conn.Do("SADD", "channels", uid)
-	conn.Do("SETNX", "channel_sortorder_"+uid, 99999)
+	updateChannelInRedis(conn, channel.UID, DefaultPrio)
 
 	return channel, nil
 }
@@ -260,8 +252,7 @@ func (b *memoryBackend) ChannelsDelete(uid string) error {
 	conn := pool.Get()
 	defer conn.Close()
 
-	conn.Do("SREM", "channels", uid)
-	conn.Do("DEL", "channel_sortorder_"+uid)
+	removeChannelFromRedis(conn, uid)
 
 	b.lock.Lock()
 	delete(b.Channels, uid)
@@ -836,3 +827,30 @@ func (b *memoryBackend) AddEventListener(el microsub.EventListener) error {
 	b.listeners = append(b.listeners, el)
 	return nil
 }
+
+func (b *memoryBackend) createChannel(name string) microsub.Channel {
+	uid := fmt.Sprintf("%012d", b.NextUid)
+	channel := microsub.Channel{
+		UID:  uid,
+		Name: name,
+	}
+	return channel
+}
+
+func (b *memoryBackend) setChannel(channel microsub.Channel) {
+	b.lock.Lock()
+	defer b.lock.Unlock()
+	b.Channels[channel.UID] = channel
+	b.Feeds[channel.UID] = []microsub.Feed{}
+	b.NextUid++
+}
+
+func updateChannelInRedis(conn redis.Conn, uid string, prio int) {
+	conn.Do("SADD", "channels", uid)
+	conn.Do("SETNX", "channel_sortorder_"+uid, prio)
+}
+
+func removeChannelFromRedis(conn redis.Conn, uid string) {
+	conn.Do("SREM", "channels", uid)
+	conn.Do("DEL", "channel_sortorder_"+uid)
+}