Add latest code and .drone.yml

This commit is contained in:
Peter Stuifzand 2018-02-16 22:13:01 +01:00
commit 6fe2eae1b6
7 changed files with 792 additions and 0 deletions

22
.drone.yml Normal file
View File

@ -0,0 +1,22 @@
pipeline:
build:
image: golang
commands:
- go get github.com/pstuifzand/microsub-server/cmd/server
- go build github.com/pstuifzand/microsub-server/cmd/server
publish:
image: plugins/docker
repo: registry.stuifzandapp.com/microsub-server
registry: registry.stuifzandapp.com
secrets: [ docker_username, docker_password ]
# deploy:
# image: appleboy/drone-ssh
# host: microsub.stuifzandapp.com
# username: hub
# secrets: ['ssh_key']
# script:
# - cd /home/hub/hub
# - docker-compose pull
# - docker-compose up -d

204
cmd/server/fetch.go Normal file
View File

@ -0,0 +1,204 @@
package main
import (
"encoding/json"
"fmt"
"log"
"net/http"
"net/url"
"os"
"strings"
"github.com/pstuifzand/microsub-server/microsub"
"willnorris.com/go/microformats"
)
var cache map[string]*microformats.Data
func init() {
cache = make(map[string]*microformats.Data)
}
func Fetch2(fetchURL string) (*microformats.Data, error) {
if !strings.HasPrefix(fetchURL, "http") {
return nil, fmt.Errorf("error parsing %s as url", fetchURL)
}
u, err := url.Parse(fetchURL)
if err != nil {
return nil, fmt.Errorf("error parsing %s as url: %s", fetchURL, err)
}
if data, e := cache[u.String()]; e {
log.Printf("HIT %s\n", u.String())
return data, nil
}
log.Printf("MISS %s\n", u.String())
resp, err := http.Get(u.String())
if err != nil {
return nil, fmt.Errorf("error while fetching %s: %s", u, err)
}
if !strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") {
return nil, fmt.Errorf("Content Type of %s = %s", fetchURL, resp.Header.Get("Content-Type"))
}
defer resp.Body.Close()
data := microformats.Parse(resp.Body, u)
cache[u.String()] = data
return data, nil
}
func Fetch(fetchURL string) []microsub.Item {
result := []microsub.Item{}
if !strings.HasPrefix(fetchURL, "http") {
return result
}
u, err := url.Parse(fetchURL)
if err != nil {
log.Printf("error parsing %s as url: %s", fetchURL, err)
return result
}
resp, err := http.Get(u.String())
if err != nil {
log.Printf("error while fetching %s: %s", u, err)
return result
}
if !strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") {
log.Printf("Content Type of %s = %s", fetchURL, resp.Header.Get("Content-Type"))
return result
}
defer resp.Body.Close()
data := microformats.Parse(resp.Body, u)
jw := json.NewEncoder(os.Stdout)
jw.SetIndent("", " ")
jw.Encode(data)
author := microsub.Author{}
for _, item := range data.Items {
if item.Type[0] == "h-feed" {
for _, child := range item.Children {
previewItem := convertMfToItem(child)
result = append(result, previewItem)
}
} else if item.Type[0] == "h-card" {
mf := item
author.Filled = true
author.Type = "card"
for prop, value := range mf.Properties {
switch prop {
case "url":
author.URL = value[0].(string)
break
case "name":
author.Name = value[0].(string)
break
case "photo":
author.Photo = value[0].(string)
break
default:
fmt.Printf("prop name not implemented for author: %s with value %#v\n", prop, value)
break
}
}
} else if item.Type[0] == "h-entry" {
previewItem := convertMfToItem(item)
result = append(result, previewItem)
}
}
for i, item := range result {
if !item.Author.Filled {
result[i].Author = author
}
}
return result
}
func convertMfToItem(mf *microformats.Microformat) microsub.Item {
item := microsub.Item{}
item.Type = mf.Type[0]
for prop, value := range mf.Properties {
switch prop {
case "published":
item.Published = value[0].(string)
break
case "url":
item.URL = value[0].(string)
break
case "name":
item.Name = value[0].(string)
break
case "latitude":
item.Latitude = value[0].(string)
break
case "longitude":
item.Longitude = value[0].(string)
break
case "like-of":
for _, v := range value {
item.LikeOf = append(item.LikeOf, v.(string))
}
break
case "bookmark-of":
for _, v := range value {
item.BookmarkOf = append(item.BookmarkOf, v.(string))
}
break
case "in-reply-to":
for _, v := range value {
item.InReplyTo = append(item.InReplyTo, v.(string))
}
break
case "summary":
if content, ok := value[0].(map[string]interface{}); ok {
item.Content.HTML = content["html"].(string)
item.Content.Text = content["value"].(string)
} else if content, ok := value[0].(string); ok {
item.Content.Text = content
}
break
case "photo":
for _, v := range value {
item.Photo = append(item.Photo, v.(string))
}
break
case "category":
for _, v := range value {
item.Category = append(item.Category, v.(string))
}
break
case "content":
if content, ok := value[0].(map[string]interface{}); ok {
item.Content.HTML = content["html"].(string)
item.Content.Text = content["value"].(string)
} else if content, ok := value[0].(string); ok {
item.Content.Text = content
}
break
default:
fmt.Printf("prop name not implemented: %s with value %#v\n", prop, value)
break
}
}
if item.Name == strings.TrimSpace(item.Content.Text) {
item.Name = ""
}
if item.Content.HTML == "" && len(item.LikeOf) > 0 {
item.Name = ""
}
fmt.Printf("%#v\n", item)
return item
}

204
cmd/server/main.go Normal file
View File

@ -0,0 +1,204 @@
package main
import (
"encoding/json"
"fmt"
"log"
"net/http"
"github.com/pstuifzand/microsub-server/microsub"
"willnorris.com/go/microformats"
)
type microsubHandler struct {
Backend microsub.Microsub
}
func simplify(item map[string][]interface{}) map[string]interface{} {
feedItem := make(map[string]interface{})
for k, v := range item {
if k == "bookmark-of" || k == "like-of" || k == "repost-of" || k == "in-reply-to" {
if value, ok := v[0].(*microformats.Microformat); ok {
m := simplify(value.Properties)
m["type"] = value.Type[0][2:]
feedItem[k] = []interface{}{m}
} else {
feedItem[k] = v
}
} else if k == "content" {
if content, ok := v[0].(map[string]interface{}); ok {
if text, e := content["value"]; e {
delete(content, "value")
if _, e := content["html"]; !e {
content["text"] = text
}
}
feedItem[k] = content
}
} else if k == "photo" {
feedItem[k] = v
} else if k == "video" {
feedItem[k] = v
} else if k == "featured" {
feedItem[k] = v
} else if value, ok := v[0].(*microformats.Microformat); ok {
m := simplify(value.Properties)
m["type"] = value.Type[0][2:]
feedItem[k] = m
} else if value, ok := v[0].(string); ok {
feedItem[k] = value
} else if value, ok := v[0].(map[string]interface{}); ok {
feedItem[k] = value
} else if value, ok := v[0].([]interface{}); ok {
feedItem[k] = value
}
}
return feedItem
}
func simplifyMicroformat(item *microformats.Microformat) map[string]interface{} {
newItem := simplify(item.Properties)
newItem["type"] = item.Type[0][2:]
children := []map[string]interface{}{}
if len(item.Children) > 0 {
for _, c := range item.Children {
child := simplifyMicroformat(c)
if c, e := child["children"]; e {
if ar, ok := c.([]map[string]interface{}); ok {
children = append(children, ar...)
}
delete(child, "children")
}
children = append(children, child)
}
newItem["children"] = children
}
return newItem
}
func simplifyMicroformatData(md *microformats.Data) []map[string]interface{} {
items := []map[string]interface{}{}
for _, item := range md.Items {
newItem := simplifyMicroformat(item)
items = append(items, newItem)
if c, e := newItem["children"]; e {
if ar, ok := c.([]map[string]interface{}); ok {
items = append(items, ar...)
}
delete(newItem, "children")
}
}
return items
}
func (h *microsubHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
fmt.Println(r.URL.String())
if r.Method == http.MethodGet {
values := r.URL.Query()
action := values.Get("action")
if action == "channels" {
channels := h.Backend.ChannelsGetList()
jw := json.NewEncoder(w)
w.Header().Add("Content-Type", "application/json")
jw.Encode(map[string][]microsub.Channel{
"channels": channels,
})
} else if action == "timeline" {
timeline := h.Backend.TimelineGet(values.Get("after"), values.Get("before"), values.Get("channel"))
jw := json.NewEncoder(w)
w.Header().Add("Content-Type", "application/json")
jw.SetIndent("", " ")
jw.Encode(timeline)
} else if action == "preview" {
md, err := Fetch2(values.Get("url"))
if err != nil {
http.Error(w, "Failed parsing url", 500)
return
}
results := simplifyMicroformatData(md)
jw := json.NewEncoder(w)
jw.SetIndent("", " ")
w.Header().Add("Content-Type", "application/json")
jw.Encode(map[string]interface{}{
"items": results,
"paging": microsub.Pagination{},
})
} else if action == "follow" {
channel := values.Get("channel")
following := h.Backend.FollowGetList(channel)
jw := json.NewEncoder(w)
w.Header().Add("Content-Type", "application/json")
jw.Encode(map[string][]microsub.Feed{
"items": following,
})
}
return
} else if r.Method == http.MethodPost {
values := r.URL.Query()
action := values.Get("action")
if action == "channels" {
name := values.Get("name")
method := values.Get("method")
uid := values.Get("channel")
if method == "delete" {
h.Backend.ChannelsDelete(uid)
w.Header().Add("Content-Type", "application/json")
fmt.Fprintln(w, "[]")
h.Backend.(Debug).Debug()
return
}
jw := json.NewEncoder(w)
if uid == "" {
channel := h.Backend.ChannelsCreate(name)
w.Header().Add("Content-Type", "application/json")
jw.Encode(channel)
} else {
channel := h.Backend.ChannelsUpdate(uid, name)
w.Header().Add("Content-Type", "application/json")
jw.Encode(channel)
}
h.Backend.(Debug).Debug()
} else if action == "follow" {
uid := values.Get("channel")
url := values.Get("url")
feed := h.Backend.FollowURL(uid, url)
w.Header().Add("Content-Type", "application/json")
jw := json.NewEncoder(w)
jw.Encode(feed)
} else if action == "unfollow" {
uid := values.Get("channel")
url := values.Get("url")
h.Backend.UnfollowURL(uid, url)
w.Header().Add("Content-Type", "application/json")
fmt.Fprintln(w, "[]")
} else if action == "search" {
query := values.Get("query")
feeds := h.Backend.Search(query)
jw := json.NewEncoder(w)
w.Header().Add("Content-Type", "application/json")
jw.Encode(map[string][]microsub.Feed{
"results": feeds,
})
}
return
}
return
}
func main() {
backend := loadMemoryBackend()
//backend := createMemoryBackend()
http.Handle("/microsub", &microsubHandler{backend})
log.Fatal(http.ListenAndServe(":80", nil))
}

209
cmd/server/memory.go Normal file
View File

@ -0,0 +1,209 @@
package main
import (
"encoding/json"
"fmt"
"os"
"strings"
"github.com/pstuifzand/microsub-server/microsub"
)
type memoryBackend struct {
Channels map[string]microsub.Channel
Feeds map[string][]microsub.Feed
//Items map[string]map[string][]microsub.Item
NextUid int
}
type Debug interface {
Debug()
}
func (b *memoryBackend) Debug() {
fmt.Println(b.Channels)
}
func (b *memoryBackend) load() {
filename := "/tmp/backend.json"
f, _ := os.Open(filename)
defer f.Close()
jw := json.NewDecoder(f)
jw.Decode(b)
}
func (b *memoryBackend) save() {
filename := "/tmp/backend.json"
f, _ := os.Create(filename)
defer f.Close()
jw := json.NewEncoder(f)
jw.Encode(b)
}
func loadMemoryBackend() microsub.Microsub {
backend := &memoryBackend{}
backend.load()
return backend
}
func createMemoryBackend() microsub.Microsub {
backend := memoryBackend{}
defer backend.save()
backend.Channels = make(map[string]microsub.Channel)
backend.Feeds = make(map[string][]microsub.Feed)
channels := []microsub.Channel{
microsub.Channel{"0000", "default"},
microsub.Channel{"0001", "notifications"},
microsub.Channel{"1000", "Friends"},
microsub.Channel{"1001", "Family"},
}
for _, c := range channels {
backend.Channels[c.UID] = c
}
backend.NextUid = 1002
return &backend
}
// ChannelsGetList gets no channels
func (b *memoryBackend) ChannelsGetList() []microsub.Channel {
channels := []microsub.Channel{}
for _, v := range b.Channels {
channels = append(channels, v)
}
return channels
}
// ChannelsCreate creates no channels
func (b *memoryBackend) ChannelsCreate(name string) microsub.Channel {
defer b.save()
uid := fmt.Sprintf("%04d", b.NextUid)
channel := microsub.Channel{
UID: uid,
Name: name,
}
b.Channels[channel.UID] = channel
b.Feeds[channel.UID] = []microsub.Feed{}
b.NextUid++
return channel
}
// ChannelsUpdate updates no channels
func (b *memoryBackend) ChannelsUpdate(uid, name string) microsub.Channel {
defer b.save()
if c, e := b.Channels[uid]; e {
c.Name = name
b.Channels[uid] = c
return c
}
return microsub.Channel{}
}
func (b *memoryBackend) ChannelsDelete(uid string) {
defer b.save()
if _, e := b.Channels[uid]; e {
delete(b.Channels, uid)
}
}
func (b *memoryBackend) TimelineGet(after, before, channel string) microsub.Timeline {
feeds := b.FollowGetList(channel)
items := []map[string]interface{}{}
for _, feed := range feeds {
md, err := Fetch2(feed.URL)
if err == nil {
results := simplifyMicroformatData(md)
found := -1
for {
for i, r := range results {
if r["type"] == "card" {
found = i
break
}
}
if found >= 0 {
card := results[found]
results = append(results[:found], results[found+1:]...)
for i := range results {
if results[i]["type"] == "entry" && results[i]["author"] == card["url"] {
results[i]["author"] = card
}
}
found = -1
continue
}
break
}
for i, r := range results {
if as, ok := r["author"].(string); ok {
if r["type"] == "entry" && strings.HasPrefix(as, "http") {
md, _ := Fetch2(as)
author := simplifyMicroformatData(md)
for _, a := range author {
if a["type"] == "card" {
results[i]["author"] = a
break
}
}
}
}
}
items = append(items, results...)
}
}
return microsub.Timeline{
Paging: microsub.Pagination{},
Items: items,
}
}
func (b *memoryBackend) FollowGetList(uid string) []microsub.Feed {
return b.Feeds[uid]
}
func (b *memoryBackend) FollowURL(uid string, url string) microsub.Feed {
defer b.save()
feed := microsub.Feed{"feed", url}
b.Feeds[uid] = append(b.Feeds[uid], feed)
return feed
}
func (b *memoryBackend) UnfollowURL(uid string, url string) {
defer b.save()
index := -1
for i, f := range b.Feeds[uid] {
if f.URL == url {
index = i
break
}
}
if index >= 0 {
feeds := b.Feeds[uid]
b.Feeds[uid] = append(feeds[:index], feeds[index+1:]...)
}
}
// TODO: improve search for feeds
func (b *memoryBackend) Search(query string) []microsub.Feed {
return []microsub.Feed{
microsub.Feed{"feed", query},
//microsub.Feed{"feed", "https://peterstuifzand.nl/rss.xml"},
}
}
func (b *memoryBackend) PreviewURL(previewUrl string) microsub.Timeline {
md, err := Fetch2(previewUrl)
if err != nil {
return microsub.Timeline{}
}
results := simplifyMicroformatData(md)
return microsub.Timeline{
Items: results,
}
}

69
cmd/server/null.go Normal file
View File

@ -0,0 +1,69 @@
package main
import (
"github.com/pstuifzand/microsub-server/microsub"
)
// NullBackend is the simplest possible backend
type NullBackend struct {
}
// ChannelsGetList gets no channels
func (b *NullBackend) ChannelsGetList() []microsub.Channel {
return []microsub.Channel{
microsub.Channel{"0000", "default"},
microsub.Channel{"0001", "notifications"},
microsub.Channel{"1000", "Friends"},
microsub.Channel{"1001", "Family"},
}
}
// ChannelsCreate creates no channels
func (b *NullBackend) ChannelsCreate(name string) microsub.Channel {
return microsub.Channel{
UID: "1234",
Name: name,
}
}
// ChannelsUpdate updates no channels
func (b *NullBackend) ChannelsUpdate(uid, name string) microsub.Channel {
return microsub.Channel{
UID: uid,
Name: name,
}
}
// ChannelsDelete delets no channels
func (b *NullBackend) ChannelsDelete(uid string) {
}
// TimelineGet gets no timeline
func (b *NullBackend) TimelineGet(after, before, channel string) microsub.Timeline {
return microsub.Timeline{
Paging: microsub.Pagination{},
Items: []map[string]interface{}{},
}
}
func (b *NullBackend) FollowGetList(uid string) []microsub.Feed {
return []microsub.Feed{}
}
func (b *NullBackend) FollowURL(uid string, url string) microsub.Feed {
return microsub.Feed{"feed", url}
}
func (b *NullBackend) UnfollowURL(uid string, url string) {
}
func (b *NullBackend) Search(query string) []microsub.Feed {
return []microsub.Feed{}
}
func (b *NullBackend) PreviewURL(url string) microsub.Timeline {
return microsub.Timeline{
Paging: microsub.Pagination{},
Items: []map[string]interface{}{},
}
}

BIN
cmd/server/server Executable file

Binary file not shown.

84
microsub/protocol.go Normal file
View File

@ -0,0 +1,84 @@
package microsub
/*
channels
search
preview
follow / unfollow
timeline
mute / unmute
block / unblock
*/
// Channel contains information about a channel.
type Channel struct {
// UID is a unique id for the channel
UID string `json:"uid"`
Name string `json:"name"`
}
type Author struct {
Filled bool `json:"-"`
Type string `json:"type"`
Name string `json:"name"`
URL string `json:"url"`
Photo string `json:"photo"`
}
type Content struct {
Text string `json:"text"`
HTML string `json:"html"`
}
// Item is a post object
type Item struct {
Type string `json:"type"`
Name string `json:"name,omitempty"`
Published string `json:"published"`
URL string `json:"url"`
Author Author `json:"author"`
Category []string `json:"category"`
Photo []string `json:"photo"`
LikeOf []string `json:"like-of"`
BookmarkOf []string `json:"bookmark-of"`
InReplyTo []string `json:"in-reply-to"`
Summary []string `json:"summary,omitempty"`
Content Content `json:"content,omitempty"`
Latitude string `json:"latitude,omitempty"`
Longitude string `json:"longitude,omitempty"`
}
// Pagination contains information about paging
type Pagination struct {
After string `json:"after,omitempty"`
Before string `json:"before,omitempty"`
}
// Timeline is a combination of items and paging information
type Timeline struct {
Items []map[string]interface{} `json:"items"`
Paging Pagination `json:"paging"`
}
type Feed struct {
Type string `json:"type"`
URL string `json:"url"`
}
// Microsub is the main protocol that should be implemented by a backend
type Microsub interface {
ChannelsGetList() []Channel
ChannelsCreate(name string) Channel
ChannelsUpdate(uid, name string) Channel
ChannelsDelete(uid string)
TimelineGet(before, after, channel string) Timeline
FollowGetList(uid string) []Feed
FollowURL(uid string, url string) Feed
UnfollowURL(uid string, url string)
Search(query string) []Feed
PreviewURL(url string) Timeline
}