README.md

@@ -6,12 +6,7 @@ a [Microsub](https://indieweb.org/Microsub) server
 
 There are two methods for installing and running ekster.
 
-### Method 1: From binaries
+### Method 1: Install ekster (from source)
-
-Download the binaries from the [latest release](https://github.com/pstuifzand/ekster/releases/) on Github.
-
-
-### Method 2: Install ekster from source with Go
 
 ekster is build using [go](https://golang.org). To be able to install ekster
 you need a Go environment. Use these commands to install the programs.
@@ -43,7 +38,7 @@ Start eksterd and pass the redis and port arguments.
 You can now access `eksterd` on port `8090`. To really use it, you should proxy
 `eksterd` behind a HTTP reverse proxy on port 80, or 443.
 
-### Method 3: Using Docker / Docker Compose
+### Method 2: Using Docker / Docker Compose
 
 It's now also possible to use docker-compose to start an ekster server. Create an empty directory. 
 Download [docker-compose.yml](https://raw.githubusercontent.com/pstuifzand/ekster/master/docker-compose.yml) from Github 

cmd/ek/main.go

@@ -314,8 +314,8 @@ func performCommands(sub microsub.Microsub, commands []string) {
 	}
 
 	if len(commands) == 2 && commands[0] == "preview" {
-		u := commands[1]
+		url := commands[1]
-		timeline, err := sub.PreviewURL(u)
+		timeline, err := sub.PreviewURL(url)
 
 		if err != nil {
 			log.Fatalf("An error occurred: %s\n", err)
@@ -338,8 +338,8 @@ func performCommands(sub microsub.Microsub, commands []string) {
 
 	if len(commands) == 3 && commands[0] == "follow" {
 		uid := commands[1]
-		u := commands[2]
+		url := commands[2]
-		_, err := sub.FollowURL(uid, u)
+		_, err := sub.FollowURL(uid, url)
 		if err != nil {
 			log.Fatalf("An error occurred: %s\n", err)
 		}
@@ -348,8 +348,8 @@ func performCommands(sub microsub.Microsub, commands []string) {
 
 	if len(commands) == 3 && commands[0] == "unfollow" {
 		uid := commands[1]
-		u := commands[2]
+		url := commands[2]
-		err := sub.UnfollowURL(uid, u)
+		err := sub.UnfollowURL(uid, url)
 		if err != nil {
 			log.Fatalf("An error occurred: %s\n", err)
 		}
@@ -359,9 +359,72 @@ func performCommands(sub microsub.Microsub, commands []string) {
 		filetype := commands[1]
 
 		if filetype == "opml" {
-			exportOpmlFromMicrosub(sub)
+			output := opml.OPML{}
+			output.Head.Title = "Microsub channels and feeds"
+			output.Head.DateCreated = time.Now().Format(time.RFC3339)
+			output.Version = "1.0"
+
+			channels, err := sub.ChannelsGetList()
+			if err != nil {
+				log.Fatalf("An error occurred: %s\n", err)
+			}
+
+			for _, c := range channels {
+				var feeds []opml.Outline
+				list, err := sub.FollowGetList(c.UID)
+				if err != nil {
+					log.Fatalf("An error occurred: %s\n", err)
+				}
+				for _, f := range list {
+					feeds = append(feeds, opml.Outline{
+						Title:   f.Name,
+						Text:    f.Name,
+						Type:    f.Type,
+						URL:     f.URL,
+						HTMLURL: f.URL,
+						XMLURL:  f.URL,
+					})
+				}
+
+				output.Body.Outlines = append(output.Body.Outlines, opml.Outline{
+					Text:     c.Name,
+					Title:    c.Name,
+					Outlines: feeds,
+				})
+			}
+
+			xml, err := output.XML()
+			if err != nil {
+				log.Fatalf("An error occurred: %s\n", err)
+			}
+			os.Stdout.WriteString(xml)
 		} else if filetype == "json" {
-			exportJsonFromMicrosub(sub)
+			contents := Export{Version: "1.0", Generator: "ek version " + Version}
+
+			channels, err := sub.ChannelsGetList()
+			if err != nil {
+				log.Fatalf("An error occurred: %s\n", err)
+			}
+
+			for _, c := range channels {
+				contents.Channels = append(contents.Channels, ExportChannel{UID: c.UID, Name: c.Name})
+			}
+
+			contents.Feeds = make(map[string][]ExportFeed)
+
+			for _, c := range channels {
+				list, err := sub.FollowGetList(c.UID)
+				if err != nil {
+					log.Fatalf("An error occurred: %s\n", err)
+				}
+				for _, f := range list {
+					contents.Feeds[c.UID] = append(contents.Feeds[c.UID], ExportFeed(f.URL))
+				}
+			}
+			err = json.NewEncoder(os.Stdout).Encode(&contents)
+			if err != nil {
+				log.Fatalf("An error occurred: %s\n", err)
+			}
 		} else {
 			log.Fatalf("unsupported filetype %q", filetype)
 		}
@@ -372,9 +435,138 @@ func performCommands(sub microsub.Microsub, commands []string) {
 		filename := commands[2]
 
 		if filetype == "opml" {
-			importOpmlIntoMicrosub(sub, filename)
+			channelMap := make(map[string]microsub.Channel)
+
+			channels, err := sub.ChannelsGetList()
+			if err != nil {
+				log.Fatalf("an error occurred: %s\n", err)
+			}
+
+			for _, c := range channels {
+				channelMap[c.Name] = c
+			}
+
+			xml, err := opml.NewOPMLFromFile(filename)
+			if err != nil {
+				log.Fatalf("An error occurred: %s\n", err)
+			}
+
+			for _, c := range xml.Body.Outlines {
+				if c.HTMLURL != "" {
+					log.Printf("First row item has url: %s\n", c.HTMLURL)
+					continue
+				}
+				if len(c.Outlines) == 0 {
+					continue
+				}
+
+				uid := ""
+
+				if ch, e := channelMap[c.Text]; !e {
+					channelCreated, err := sub.ChannelsCreate(c.Text)
+					if err != nil {
+						log.Printf("An error occurred: %q\n", err)
+						continue
+					}
+
+					uid = channelCreated.UID
+					log.Printf("Channel created: %s\n", c.Text)
+				} else {
+					uid = ch.UID
+				}
+
+				feedMap := make(map[string]bool)
+
+				feeds, err := sub.FollowGetList(uid)
+				if err != nil {
+					log.Fatalf("An error occurred: %q\n", err)
+				}
+
+				for _, f := range feeds {
+					feedMap[f.URL] = true
+				}
+
+				for _, f := range c.Outlines {
+					if f.HTMLURL == "" {
+						log.Println("Missing url on second row item")
+						continue
+					}
+
+					if _, e := feedMap[f.HTMLURL]; !e {
+						_, err := sub.FollowURL(uid, f.HTMLURL)
+						if err != nil {
+							log.Printf("An error occurred: %q\n", err)
+							continue
+						}
+						log.Printf("Feed followed: %s\n", f.HTMLURL)
+					}
+				}
+			}
 		} else if filetype == "json" {
-			importJsonIntoMicrosub(sub, filename)
+			var export Export
+
+			f, err := os.Open(filename)
+			if err != nil {
+				log.Fatalf("can't open file %s: %s", filename, err)
+			}
+			defer f.Close()
+
+			err = json.NewDecoder(f).Decode(&export)
+			if err != nil {
+				log.Fatalf("error while reading %s: %s", filename, err)
+			}
+
+			channelMap := make(map[string]microsub.Channel)
+
+			channels, err := sub.ChannelsGetList()
+			if err != nil {
+				log.Fatalf("an error occurred: %s\n", err)
+			}
+
+			for _, c := range channels {
+				channelMap[c.Name] = c
+			}
+
+			for _, c := range export.Channels {
+				uid := ""
+
+				if ch, e := channelMap[c.Name]; !e {
+					channelCreated, err := sub.ChannelsCreate(c.Name)
+					if err != nil {
+						log.Printf("An error occurred: %q\n", err)
+						continue
+					}
+
+					uid = channelCreated.UID
+					log.Printf("Channel created: %s\n", c.Name)
+				} else {
+					uid = ch.UID
+				}
+
+				feedMap := make(map[string]bool)
+
+				feeds, err := sub.FollowGetList(uid)
+				if err != nil {
+					log.Fatalf("An error occurred: %q\n", err)
+				}
+
+				for _, f := range feeds {
+					feedMap[f.URL] = true
+				}
+
+				for _, feed := range export.Feeds[uid] {
+
+					if _, e := feedMap[string(feed)]; !e {
+						_, err := sub.FollowURL(uid, string(feed))
+						if err != nil {
+							log.Printf("An error occurred: %q\n", err)
+							continue
+						}
+						log.Printf("Feed followed: %s\n", string(feed))
+					}
+				}
+			}
+
 		} else {
 			log.Fatalf("unsupported filetype %q", filetype)
 		}
@@ -385,196 +577,6 @@ func performCommands(sub microsub.Microsub, commands []string) {
 	}
 }
 
-func exportOpmlFromMicrosub(sub microsub.Microsub) {
-	output := opml.OPML{}
-	output.Head.Title = "Microsub channels and feeds"
-	output.Head.DateCreated = time.Now().Format(time.RFC3339)
-	output.Version = "1.0"
-	channels, err := sub.ChannelsGetList()
-	if err != nil {
-		log.Fatalf("An error occurred: %s\n", err)
-	}
-	for _, c := range channels {
-		var feeds []opml.Outline
-		list, err := sub.FollowGetList(c.UID)
-		if err != nil {
-			log.Fatalf("An error occurred: %s\n", err)
-		}
-		for _, f := range list {
-			feeds = append(feeds, opml.Outline{
-				Title:   f.Name,
-				Text:    f.Name,
-				Type:    f.Type,
-				URL:     f.URL,
-				HTMLURL: f.URL,
-				XMLURL:  f.URL,
-			})
-		}
-
-		output.Body.Outlines = append(output.Body.Outlines, opml.Outline{
-			Text:     c.Name,
-			Title:    c.Name,
-			Outlines: feeds,
-		})
-	}
-	xml, err := output.XML()
-	if err != nil {
-		log.Fatalf("An error occurred: %s\n", err)
-	}
-	os.Stdout.WriteString(xml)
-}
-
-func exportJsonFromMicrosub(sub microsub.Microsub) {
-	contents := Export{Version: "1.0", Generator: "ek version " + Version}
-	channels, err := sub.ChannelsGetList()
-	if err != nil {
-		log.Fatalf("An error occurred: %s\n", err)
-	}
-	for _, c := range channels {
-		contents.Channels = append(contents.Channels, ExportChannel{UID: c.UID, Name: c.Name})
-	}
-	contents.Feeds = make(map[string][]ExportFeed)
-	for _, c := range channels {
-		list, err := sub.FollowGetList(c.UID)
-		if err != nil {
-			log.Fatalf("An error occurred: %s\n", err)
-		}
-		for _, f := range list {
-			contents.Feeds[c.UID] = append(contents.Feeds[c.UID], ExportFeed(f.URL))
-		}
-	}
-	err = json.NewEncoder(os.Stdout).Encode(&contents)
-	if err != nil {
-		log.Fatalf("An error occurred: %s\n", err)
-	}
-}
-
-func importJsonIntoMicrosub(sub microsub.Microsub, filename string) {
-	var export Export
-	f, err := os.Open(filename)
-	if err != nil {
-		log.Fatalf("can't open file %s: %s", filename, err)
-	}
-	defer f.Close()
-	err = json.NewDecoder(f).Decode(&export)
-	if err != nil {
-		log.Fatalf("error while reading %s: %s", filename, err)
-	}
-	channelMap := make(map[string]microsub.Channel)
-	channels, err := sub.ChannelsGetList()
-	if err != nil {
-		log.Fatalf("an error occurred: %s\n", err)
-	}
-	for _, c := range channels {
-		channelMap[c.Name] = c
-	}
-	for _, c := range export.Channels {
-		uid := ""
-
-		if ch, e := channelMap[c.Name]; !e {
-			channelCreated, err := sub.ChannelsCreate(c.Name)
-			if err != nil {
-				log.Printf("An error occurred: %q\n", err)
-				continue
-			}
-
-			uid = channelCreated.UID
-			log.Printf("Channel created: %s\n", c.Name)
-		} else {
-			uid = ch.UID
-		}
-
-		feedMap := make(map[string]bool)
-
-		feeds, err := sub.FollowGetList(uid)
-		if err != nil {
-			log.Fatalf("An error occurred: %q\n", err)
-		}
-
-		for _, f := range feeds {
-			feedMap[f.URL] = true
-		}
-
-		for _, feed := range export.Feeds[uid] {
-
-			if _, e := feedMap[string(feed)]; !e {
-				_, err := sub.FollowURL(uid, string(feed))
-				if err != nil {
-					log.Printf("An error occurred: %q\n", err)
-					continue
-				}
-				log.Printf("Feed followed: %s\n", string(feed))
-			}
-		}
-	}
-}
-
-func importOpmlIntoMicrosub(sub microsub.Microsub, filename string) {
-	channelMap := make(map[string]microsub.Channel)
-	channels, err := sub.ChannelsGetList()
-	if err != nil {
-		log.Fatalf("an error occurred: %s\n", err)
-	}
-	for _, c := range channels {
-		channelMap[c.Name] = c
-	}
-	xml, err := opml.NewOPMLFromFile(filename)
-	if err != nil {
-		log.Fatalf("An error occurred: %s\n", err)
-	}
-	for _, c := range xml.Body.Outlines {
-		if c.HTMLURL != "" {
-			log.Printf("First row item has url: %s\n", c.HTMLURL)
-			continue
-		}
-		if len(c.Outlines) == 0 {
-			continue
-		}
-
-		uid := ""
-
-		if ch, e := channelMap[c.Text]; !e {
-			channelCreated, err := sub.ChannelsCreate(c.Text)
-			if err != nil {
-				log.Printf("An error occurred: %q\n", err)
-				continue
-			}
-
-			uid = channelCreated.UID
-			log.Printf("Channel created: %s\n", c.Text)
-		} else {
-			uid = ch.UID
-		}
-
-		feedMap := make(map[string]bool)
-
-		feeds, err := sub.FollowGetList(uid)
-		if err != nil {
-			log.Fatalf("An error occurred: %q\n", err)
-		}
-
-		for _, f := range feeds {
-			feedMap[f.URL] = true
-		}
-
-		for _, f := range c.Outlines {
-			if f.HTMLURL == "" {
-				log.Println("Missing url on second row item")
-				continue
-			}
-
-			if _, e := feedMap[f.HTMLURL]; !e {
-				_, err := sub.FollowURL(uid, f.HTMLURL)
-				if err != nil {
-					log.Printf("An error occurred: %q\n", err)
-					continue
-				}
-				log.Printf("Feed followed: %s\n", f.HTMLURL)
-			}
-		}
-	}
-}
-
 func showItem(item *microsub.Item) {
 	if item.Name != "" {
 		fmt.Printf("%s - ", item.Name)

cmd/eksterd/main.go

@@ -114,11 +114,6 @@ func main() {
 
 	if createBackend {
 		backend = createMemoryBackend()
-
-		log.Println(`Config file "backend.json" is created in the current directory.`)
-		log.Println(`Update "Me" variable to your website address "https://example.com/"`)
-		log.Println(`Update "TokenEndpoint" variable to the address of your token endpoint "https://example.com/token"`)
-
 		return
 	}
 

cmd/eksterd/memory.go

@@ -41,8 +41,6 @@ import (
 	"willnorris.com/go/microformats"
 )
 
-const DefaultPrio = 9999999
-
 type memoryBackend struct {
 	hubIncomingBackend
 
@@ -110,24 +108,23 @@ func (b *memoryBackend) load() error {
 		return err
 	}
 
-	return nil
-}
-
-func (b *memoryBackend) refreshChannels() {
 	conn := pool.Get()
 	defer conn.Close()
 
+	conn.Do("SETNX", "channel_sortorder_notifications", 1)
+
 	conn.Do("DEL", "channels")
 
-	updateChannelInRedis(conn, "notifications", 1)
-
 	b.lock.RLock()
+	defer b.lock.RUnlock()
+
 	for uid, channel := range b.Channels {
 		log.Printf("loading channel %s - %s\n", uid, channel.Name)
-		updateChannelInRedis(conn, channel.UID, DefaultPrio)
+		conn.Do("SADD", "channels", uid)
+		conn.Do("SETNX", "channel_sortorder_"+uid, 99999)
 	}
 
-	b.lock.RUnlock()
+	return nil
 }
 
 func (b *memoryBackend) save() {
@@ -148,7 +145,6 @@ func loadMemoryBackend() microsub.Microsub {
 		log.Printf("Error while loadingbackend: %v\n", err)
 		return nil
 	}
-	backend.refreshChannels()
 
 	return backend
 }
@@ -157,23 +153,25 @@ func createMemoryBackend() microsub.Microsub {
 	backend := memoryBackend{}
 	backend.lock.Lock()
 
+	backend.Channels = make(map[string]microsub.Channel)
 	backend.Feeds = make(map[string][]microsub.Feed)
 	channels := []microsub.Channel{
 		{UID: "notifications", Name: "Notifications"},
 		{UID: "home", Name: "Home"},
 	}
-
-	backend.Channels = make(map[string]microsub.Channel)
 	for _, c := range channels {
 		backend.Channels[c.UID] = c
 	}
-
 	backend.NextUid = 1000000
 	backend.Me = "https://example.com/"
 
 	backend.lock.Unlock()
 
 	backend.save()
+
+	log.Println(`Config file "backend.json" is created in the current directory.`)
+	log.Println(`Update "Me" variable to your website address "https://example.com/"`)
+	log.Println(`Update "TokenEndpoint" variable to the address of your token endpoint "https://example.com/token"`)
 	return &backend
 }
 
@@ -210,13 +208,23 @@ func (b *memoryBackend) ChannelsGetList() ([]microsub.Channel, error) {
 func (b *memoryBackend) ChannelsCreate(name string) (microsub.Channel, error) {
 	defer b.save()
 
-	channel := b.createChannel(name)
-	b.setChannel(channel)
-
 	conn := pool.Get()
 	defer conn.Close()
 
-	updateChannelInRedis(conn, channel.UID, DefaultPrio)
+	uid := fmt.Sprintf("%04d", b.NextUid)
+	channel := microsub.Channel{
+		UID:  uid,
+		Name: name,
+	}
+
+	b.lock.Lock()
+	b.Channels[channel.UID] = channel
+	b.Feeds[channel.UID] = []microsub.Feed{}
+	b.NextUid++
+	b.lock.Unlock()
+
+	conn.Do("SADD", "channels", uid)
+	conn.Do("SETNX", "channel_sortorder_"+uid, 99999)
 
 	return channel, nil
 }
@@ -252,7 +260,8 @@ func (b *memoryBackend) ChannelsDelete(uid string) error {
 	conn := pool.Get()
 	defer conn.Close()
 
-	removeChannelFromRedis(conn, uid)
+	conn.Do("SREM", "channels", uid)
+	conn.Do("DEL", "channel_sortorder_"+uid)
 
 	b.lock.Lock()
 	delete(b.Channels, uid)
@@ -827,30 +836,3 @@ func (b *memoryBackend) AddEventListener(el microsub.EventListener) error {
 	b.listeners = append(b.listeners, el)
 	return nil
 }
-
-func (b *memoryBackend) createChannel(name string) microsub.Channel {
-	uid := fmt.Sprintf("%012d", b.NextUid)
-	channel := microsub.Channel{
-		UID:  uid,
-		Name: name,
-	}
-	return channel
-}
-
-func (b *memoryBackend) setChannel(channel microsub.Channel) {
-	b.lock.Lock()
-	defer b.lock.Unlock()
-	b.Channels[channel.UID] = channel
-	b.Feeds[channel.UID] = []microsub.Feed{}
-	b.NextUid++
-}
-
-func updateChannelInRedis(conn redis.Conn, uid string, prio int) {
-	conn.Do("SADD", "channels", uid)
-	conn.Do("SETNX", "channel_sortorder_"+uid, prio)
-}
-
-func removeChannelFromRedis(conn redis.Conn, uid string) {
-	conn.Do("SREM", "channels", uid)
-	conn.Do("DEL", "channel_sortorder_"+uid)
-}