Problem: indexing search objects is slow

Solution: batch indexing search objects

.drone.yml

@@ -37,8 +37,10 @@ steps:
     commands:
       - cd list-editor
       - npm install
+      - npm test
       - cd ../editor
       - npm install
+      - npm test
       - npm run docker
 
   - name: rebuild-cache-with-filesystem

.pre-commit-config.yaml

@@ -19,3 +19,11 @@ repos:
       language: system
       entry: drone lint --trusted
       files: .drone.yml
+    - id: npm-test-list-editor
+      name: npm test list editor
+      language: system
+      entry: bash -c "cd list-editor && npm test"
+    - id: npm-test-editor
+      name: npm test editor
+      language: system
+      entry: bash -c "cd editor && npm test"

block.go

@@ -0,0 +1,92 @@
+/*
+ * Wiki - A wiki with editor
+ * Copyright (c) 2021-2021 Peter Stuifzand
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+type BlockRepository interface {
+	Save(id string, block Block) error
+	Load(id string) (Block, error)
+}
+
+type blockSaveMessage struct {
+	id    string
+	block Block
+}
+
+type blockRepo struct {
+	dirname string
+	saveC   chan blockSaveMessage
+	errC    chan error
+}
+
+func saveBlock(dirname, id string, block Block) error {
+	f, err := os.OpenFile(filepath.Join(dirname, BlocksDirectory, id), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	enc := json.NewEncoder(f)
+	enc.SetIndent("", "  ")
+	err = enc.Encode(&block)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func NewBlockRepo(dirname string) BlockRepository {
+	saveC := make(chan blockSaveMessage, 1)
+	errC := make(chan error)
+	go func() {
+		for msg := range saveC {
+			err := saveBlock(dirname, msg.id, msg.block)
+			errC <- err
+		}
+	}()
+	return &blockRepo{
+		dirname: dirname,
+		saveC:   saveC,
+		errC:    errC,
+	}
+}
+
+func (br *blockRepo) Save(id string, block Block) error {
+	br.saveC <- blockSaveMessage{id, block}
+	err := <-br.errC
+	return err
+}
+
+func (br *blockRepo) Load(id string) (Block, error) {
+	f, err := os.Open(filepath.Join(br.dirname, BlocksDirectory, id))
+	if err != nil {
+		return Block{}, fmt.Errorf("%q: %w", id, BlockNotFound)
+	}
+	defer f.Close()
+	var block Block
+	err = json.NewDecoder(f).Decode(&block)
+	if err != nil {
+		return Block{}, fmt.Errorf("%q: %w", id, err)
+	}
+	return block, nil
+}

editor/package-lock.json

@@ -1895,6 +1895,14 @@
       "integrity": "sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg==",
       "dev": true
     },
+    "chrono-node": {
+      "version": "2.3.5",
+      "resolved": "https://registry.npmjs.org/chrono-node/-/chrono-node-2.3.5.tgz",
+      "integrity": "sha512-QIWEgXYVn55/Nsgdqbe6inqW+GoK3B6Qtga8AWdpq+nd+mOZVMxa+SGwPq/XjY+nKN+toQGu8KifCPwUkmz2sg==",
+      "requires": {
+        "dayjs": "^1.10.0"
+      }
+    },
     "class-utils": {
       "version": "0.3.6",
       "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz",
@@ -2655,6 +2663,11 @@
         "assert-plus": "^1.0.0"
       }
     },
+    "dayjs": {
+      "version": "1.10.7",
+      "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.10.7.tgz",
+      "integrity": "sha512-P6twpd70BcPK34K26uJ1KT3wlhpuOAPoMwJzpsIWUxHZ7wpmbdZL/hQqBDfz7hGurYSa5PhzdhDHtt319hL3ig=="
+    },
     "debug": {
       "version": "2.6.9",
       "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",

editor/package.json

@@ -8,6 +8,7 @@
     "@egjs/hammerjs": "^2.0.17",
     "axios": "^0.19.2",
     "bulma": "^0.7.5",
+    "chrono-node": "^2.3.5",
     "clipboard": "^2.0.8",
     "copy-text-to-clipboard": "^2.2.0",
     "css-loader": "^3.6.0",
@@ -59,7 +60,7 @@
   },
   "scripts": {
     "test": "node_modules/.bin/mocha -r esm",
-    "watch": "webpack --watch",
+    "watch": "webpack --watch --mode=development",
     "start": "webpack-dev-server --open --hot",
     "build": "webpack --progress --mode=production",
     "docker": "webpack --no-color --mode=production"

editor/src/editor.js

@@ -36,7 +36,7 @@ function addSaver(editor, saveUrl, page, beforeSave) {
         save() {
             return editor.save()
                 .then(outputData => {
-                    beforeSave()
+                    beforeSave(outputData)
                     let data = {
                         'json': 1,
                         'p': page,
@@ -108,18 +108,12 @@ function showSearchResultsExtended(element, template, searchTool, query, input,
     return searchTool(query).then(results => {
         let opt = options || {};
         if (opt.showOnlyResults && (query.length === 0 || !results.length)) {
-            $lc.fadeOut()
+            $lc.hide()
             return
         }
         $lc.data('result-type', resultType)
 
-        if (opt.belowCursor) {
+        const visible = $(':visible', $lc).length
-            let pos = getCaretCoordinates(input, value.selectionEnd, {})
-            let off = $(input).offset()
-            pos.top += off.top + pos.height
-            pos.left += off.left
-            $lc.offset(pos)
-        }
 
         let selectedPos = 0;
         let selected = $lc.find('li.selected');
@@ -127,6 +121,8 @@ function showSearchResultsExtended(element, template, searchTool, query, input,
             selectedPos = $lc.find('li').index(selected[0])
         }
 
+        const isEditing = window.location.pathname.match(/^\/edit\//)
+
         let $ul = el('ul',
             _.map(results, (hit, i) => {
                 let div = el('div', []);
@@ -139,7 +135,7 @@ function showSearchResultsExtended(element, template, searchTool, query, input,
                 ];
                 if (hit.ref) {
                     children = hit.ref ? [el('a', children)] : children;
-                    children[0].setAttribute('href', '/edit/' + hit.ref)
+                    children[0].setAttribute('href', (isEditing ? '/edit/' : '/') + hit.ref)
                 }
                 const li = el('li', children)
                 if (selectedPos === i) li.classList.add('selected')
@@ -148,10 +144,17 @@ function showSearchResultsExtended(element, template, searchTool, query, input,
             })
         )
 
-        $lc.html($ul).fadeIn()
+        $lc.show()
+            .html($ul)
+
+        if (opt.belowCursor) {
+            let pos = getCaretCoordinates(input, value.selectionEnd, {})
+            let off = $(input).offset()
+            $lc.offset({top: off.top + pos.top + pos.height, left: off.left + pos.left})
+        }
 
         return results
-    })
+    }).catch(e => console.log('searchtool', e))
 }
 
 function formatLineResult(hits, key) {
@@ -175,6 +178,17 @@ function formatLineResult(hits, key) {
     ]
 }
 
+function formatLineWithoutTitleResult(hit, key) {
+    if (hit.line.match(/query[!@]?:/)) return []
+    return [{
+        text: hit.line,
+        indented: 0,
+        fold: 'open',
+        hidden: false,
+        fleeting: true
+    }]
+}
+
 function formatTitleResult(hit) {
     return [
         {
@@ -283,7 +297,7 @@ function Editor(holder, input) {
                                             return td
                                         })
                                     ])
-                                })
+                                }).catch(e => console.log('while fetching metakv', e))
                         })
                     })
                     .then(mflatten)
@@ -312,8 +326,9 @@ function Editor(holder, input) {
                         div.classList.add('table-wrapper')
 
                         return element.html(div);
+                    }).catch(e => console.log('while creating table', e))
             })
-            })
+            .catch(e => console.log('transformTable', e))
     }
 
     async function transformMathExpression(converted, scope) {
@@ -349,13 +364,12 @@ function Editor(holder, input) {
         }
 
         let converted = text
-        let todo;
 
         if (converted === '{{table}}') {
             transformTable.call(this, editor, id, element);
             return
         } else if (converted.startsWith("```", 0) || converted.startsWith("$$", 0)) {
-            converted = MD.render(converted)
+            converted = MD.renderInline(converted)
         } else if (converted.startsWith("=", 0)) {
             transformMathExpression(converted, scope)
                 .then(converted => {
@@ -367,31 +381,36 @@ function Editor(holder, input) {
                 })
                 .catch(e => console.warn(e))
         } else {
-            let re = /^([A-Z0-9 ]+)::\s*(.*)$/i;
+            // let re = /^([A-Z0-9 ]+)::\s*(.*)$/i;
-            let res = text.match(re)
+            // let res = text.match(re)
-            if (res) {
+            // if (res) {
-                converted = '<span class="metadata-key">[[' + res[1] + ']]</span>'
+            //     converted = '<span class="metadata-key">[[' + res[1] + ']]</span>'
-                if (res[2]) {
+            //     if (res[2]) {
-                    converted += ': ' + res[2]
+            //         converted += ': ' + res[2]
-                }
+            //     }
-            } else if (text.match(/#\[\[TODO]]/)) {
+            // } else if (text.match(/#\[\[TODO]]/)) {
-                converted = converted.replace('#[[TODO]]', '<input class="checkbox" type="checkbox" />')
+            //     converted = converted.replace('#[[TODO]]', '<input class="checkbox" type="checkbox" />')
-                todo = true;
+            //     todo = true;
-            } else if (text.match(/#\[\[DONE]]/)) {
+            // } else if (text.match(/#\[\[DONE]]/)) {
-                converted = converted.replace('#[[DONE]]', '<input class="checkbox" type="checkbox" checked />')
+            //     converted = converted.replace('#[[DONE]]', '<input class="checkbox" type="checkbox" checked />')
-                todo = false;
+            //     todo = false;
-            }
+            // }
-            MD.options.html = true
+            converted = MD.render(converted)
-            converted = MD.renderInline(converted)
-            MD.options.html = false
         }
 
-        if (todo !== undefined) {
+        try {
+            const todoItem = $.parseHTML(converted)
+            if (todoItem.length && $(todoItem[0]).is(':checkbox')) {
+                const todo = !$(todoItem[0]).is(':checked')
                 element.toggleClass('todo--done', todo === false)
                 element.toggleClass('todo--todo', todo === true)
             } else {
                 element.removeClass(['todo--todo', 'todo--done'])
             }
+        } catch (e) {
+            // problem with $(converted) is that it could be treated as a jQuery selector expression instead of normal text
+            // the wiki text is not quite like selectors
+        }
 
         element.html(converted)
     }
@@ -428,10 +447,16 @@ function Editor(holder, input) {
         let saveUrl = element.dataset.saveurl;
         let page = element.dataset.page;
 
+        let beforeSave = (curDoc) => {
+            indicator.setText('saving...')
+        }
+
         addIndicator(
-            addSaver(editor, saveUrl, page, () => indicator.setText('saving...')),
+            addSaver(editor, saveUrl, page, beforeSave),
             indicator
-        ).save().then(() => indicator.done())
+        ).save()
+            .then(() => indicator.done())
+            .catch(e => console.log('editor.change', e))
     })
 
 // editor.on('rendered', function () {
@@ -514,13 +539,13 @@ function Editor(holder, input) {
                 const isVisible = $('#link-complete:visible').length > 0;
 
                 if (event.key === 'Escape' && isVisible) {
-                    $lc.fadeOut()
+                    $lc.hide()
                     return false
                 } else if (event.key === 'Enter' && isVisible) {
                     const element = $lc.find('li.selected')
                     const linkName = element.text()
                     $lc.trigger('popup:selected', [linkName, $lc.data('result-type'), element])
-                    $lc.fadeOut()
+                    $lc.hide()
                     return false
                 } else if (event.key === 'ArrowUp' && isVisible) {
                     const selected = $lc.find('li.selected')
@@ -580,7 +605,7 @@ function Editor(holder, input) {
                 if (event.key === '/') {
                     searchEnabled = true
                 }
-                if (searchEnabled && event.key === 'Escape') {
+                if (searchEnabled && (event.key === 'Escape' || event.key === 'Space')) {
                     searchEnabled = false
                     return false;
                 }
@@ -607,18 +632,19 @@ function Editor(holder, input) {
 
                 if (insideSearch) {
                     let query = value.substring(start + 1, end);
-                    showSearchResults(commandSearch, query, input, value, 'command').then(results => {
+                    showSearchResults(commandSearch, query, input, value, 'command')
-                        if (query.length > 0 && results.length === 0) {
+                        .then(results => {
+                            if (query.length > 0 && (!results || results.length === 0)) {
                                 searchEnabled = false
                             }
-                    })
+                        }).catch(e => console.log('showSearchResults', e))
                     return true
                 } else if (insideLink) {
                     let query = value.substring(start, end);
                     showSearchResults(titleSearch, query, input, value, 'link');
                     return true
                 } else {
-                    $('#link-complete').fadeOut();
+                    $('#link-complete').hide();
                 }
             })
         })
@@ -634,20 +660,30 @@ function Editor(holder, input) {
 
             let query = $input.val()
 
-            match(query, /{{query(!?):\s*([^}]+)}}/)
+            match(query, /{{query([!@]?):\s*([^}]+)}}/)
                 .then(res => {
-                    if (res[1] === '!') {
+                    if (res[1] === '@') {
-                        return search.startQuery(res[2])
+                        return search.startQuery(res[2], {internal: false})
+                            .then(hits => _.flatMap(hits, formatLineWithoutTitleResult))
+                            .then(results => editor.replaceChildren(id, results))
+                            .catch(e => console.log('search query', e))
+                            .finally(() => editor.render())
+                    } else if (res[1] === '!') {
+                        return search.startQuery(res[2], {internal: false})
                             .then(hits => _.uniqBy(_.flatMap(hits, formatTitleResult), _.property('text')))
                             .then(results => editor.replaceChildren(id, results))
+                            .catch(e => console.log('search query', e))
                             .finally(() => editor.render())
                     } else {
-                        return search.startQuery(res[2])
+                        return search.startQuery(res[2], {internal: false})
                             .then(hits => _.groupBy(hits, (it) => it.title))
                             .then(hits => _.flatMap(hits, formatLineResult))
                             .then(results => editor.replaceChildren(id, results))
+                            .catch(e => console.log('search query', e))
                             .finally(() => editor.render())
                     }
+                }).catch(e => {
+                    console.log(e)
                 })
         });
         return editor
@@ -658,7 +694,7 @@ function match(s, re) {
     return new Promise((resolve, reject) => {
         let res = s.match(re)
         if (res) resolve(res)
-        else reject()
+        else reject(s + ' did not match ' + re)
     });
 }
 

editor/src/fuse.js

@@ -1,4 +1,6 @@
 import Fuse from 'fuse.js'
+import * as chrono from "chrono-node";
+import moment from "moment";
 
 function createTitleSearch() {
     return new Promise(function (resolve, reject) {
@@ -29,6 +31,12 @@ function createTitleSearch() {
                     titleSearch: query => {
                         return new Promise((resolve, reject) => {
                             let search = titleFuse.search(query);
+                            let parseResult = chrono.nl.casual.parse(query)
+                            if (parseResult.length) {
+                                let m = moment(parseResult[0].start.date())
+                                const title = m.format('LL')
+                                search.unshift({item: {title, label: "Suggested page '" + title + "'"}})
+                            }
                             search.unshift({item: {title: query, label: "Create page '" + query + "'"}})
                             search = search.slice(0, 25)
                             resolve(search)

editor/src/index.js

@@ -3,16 +3,17 @@ import moment from 'moment'
 import './styles.scss'
 import Editor from './editor'
 import MD from './markdown'
-import wikiGraph from "./graph";
+// import wikiGraph from "./graph";
+import "./sr";
 
 moment.locale('nl')
 // mermaid.initialize({startOnLoad: true})
 // PrismJS.plugins.filterHighlightAll.reject.addSelector('.language-mermaid')
 // PrismJS.plugins.filterHighlightAll.reject.addSelector('.language-dot')
 
-$(function () {
+// $(function () {
-    setTimeout(() => wikiGraph('.graph-network'), 1);
+//     setTimeout(() => wikiGraph('.graph-network'), 1);
-})
+// })
 
 /*
  * EVENTS
@@ -34,7 +35,7 @@ $(document).on('keydown', '.keyboard-list', function (event) {
 $(document).on('keydown', '#search-input', function (event) {
     let $ac = $('#autocomplete:visible');
     if (event.key === 'Escape') {
-        $(this).val('');
+        $(this).val('').removeClass('is-error');
 
         if ($ac.length) {
             $ac.fadeOut();
@@ -95,3 +96,25 @@ $(document).on('click', '.calendar-update', function (event) {
         })
     return false
 })
+
+function activateTab(tab) {
+    console.log(tab)
+    $('.tabs .tab-page').toggleClass('tab-active', false);
+    $($(tab).data('target')).toggleClass('tab-active', true);
+    $('.tab-bar .tab').toggleClass('tab-active', false)
+    $(tab).toggleClass('tab-active', true)
+}
+
+$(document).on('click', '.tab', function () {
+    activateTab(this)
+});
+
+$(function () {
+    activateTab($('.tab-bar .tab-active')[0]);
+
+    $('.navbar-burger').on('click', function () {
+        let open = $(this).hasClass('is-active')
+        $('#' + $(this).data('target')).toggleClass('is-active', !open)
+        $(this).toggleClass('is-active', !open)
+    });
+});

editor/src/markdown-tag.js

@@ -0,0 +1,56 @@
+'use strict'
+
+var util = require('util')
+
+function Plugin(options) {
+    var self = function (md) {
+        self.options = options
+        self.init(md)
+    }
+
+    self.__proto__ = Plugin.prototype
+    self.id = 'markdown-tag'
+
+    return self
+}
+
+util.inherits(Plugin, Function)
+
+Plugin.prototype.init = function (md) {
+    md.inline.ruler.push(this.id, this.parse.bind(this))
+    md.renderer.rules[this.id] = this.render.bind(this)
+}
+
+export function tagParser(id, state, silent) {
+    let input = state.src.slice(state.pos);
+
+    const match = /^#\S+/.exec(input)
+    if (!match) {
+        return false
+    }
+
+    console.log(match)
+    const prefixLength = match[0].length
+    if (!silent) {
+        console.log(state.src, state.pos, prefixLength)
+        let link = state.src.slice(state.pos + 1, state.pos + prefixLength)
+        let token = state.push(id, '', 0)
+        token.meta = {match: link, tag: true}
+        console.log(token)
+    }
+    state.pos += prefixLength
+    return true
+}
+
+Plugin.prototype.parse = function (state, silent) {
+    return tagParser(this.id, state, silent)
+}
+
+Plugin.prototype.render = function (tokens, id, options, env) {
+    let {match: link} = tokens[id].meta
+    return '<a href="' + this.options.relativeBaseURL + encodeURIComponent(link) + '" class="tag">' + '#' + link + '</a>';
+}
+
+export default (options) => {
+    return Plugin(options);
+}

editor/src/markdown.js

@@ -1,7 +1,9 @@
 import MarkdownIt from "markdown-it";
-import MarkdownItWikilinks from "./wikilinks";
+import MarkdownItWikilinks2 from "./wikilinks2";
+import MarkdownItMetadata from "./metadatalinks";
 import MarkdownItMark from "markdown-it-mark";
 import MarkdownItKatex from "markdown-it-katex";
+import MarkdownItTag from "./markdown-tag";
 
 const MD = new MarkdownIt({
     linkify: true,
@@ -12,14 +14,26 @@ const MD = new MarkdownIt({
         return '';
     }
 })
-
+MD.use(MarkdownItWikilinks2({
-MD.use(MarkdownItWikilinks({
     baseURL: document.querySelector('body').dataset.baseUrl,
     uriSuffix: '',
     relativeBaseURL: '/edit/',
     htmlAttributes: {
         class: 'wiki-link'
     },
-})).use(MarkdownItMark).use(MarkdownItKatex)
+}))
+MD.use(MarkdownItMetadata())
+// MD.use(MarkdownItWikilinks({
+//     baseURL: document.querySelector('body').dataset.baseUrl,
+//     uriSuffix: '',
+//     relativeBaseURL: '/edit/',
+//     htmlAttributes: {
+//         class: 'wiki-link'
+//     },
+// }))
+MD.use(MarkdownItMark).use(MarkdownItKatex)
+MD.use(MarkdownItTag({
+    relativeBaseURL: '/edit/',
+}))
 
 export default MD;

editor/src/menu.js

@@ -1,6 +1,8 @@
 import $ from 'jquery'
 import 'jquery-contextmenu'
 import copy from 'copy-text-to-clipboard'
+import axios from "axios";
+import qs from "querystring";
 
 function renderTree(tree) {
     if (!tree) return []
@@ -22,7 +24,17 @@ function connectContextMenu(editor) {
             createNewPage: {
                 name: 'Create page from item',
                 callback: function (key, opt) {
-                    console.log('Create page from item', key, opt)
+                    console.log('Create page from item')
+                    editor.flat(this, {base: true}).then(result => {
+                        let data = {
+                            'json': 1,
+                            'p': result.title,
+                            'summary': "",
+                            'content': JSON.stringify(result.children),
+                        };
+                        console.log(data)
+                        return axios.post('/save/', qs.encode(data))
+                    }).then()
                 },
                 className: 'action-new-page'
             },

editor/src/metadatalinks.js

@@ -0,0 +1,54 @@
+'use strict'
+
+var util = require('util')
+
+function Plugin(options) {
+    var self = function (md) {
+        self.options = options
+        self.init(md)
+    }
+
+    self.__proto__ = Plugin.prototype
+    self.id = 'metadata'
+
+    return self
+}
+
+util.inherits(Plugin, Function)
+
+Plugin.prototype.init = function (md) {
+    md.inline.ruler.before('text', this.id, this.parse.bind(this), {})
+    md.renderer.rules[this.id] = this.render.bind(this)
+}
+
+export function metadataLinkParser(id, state, silent) {
+    let input = state.src;
+    if (state.pos === 0 && input.match(/^([A-Za-z0-9 ]+)::/)) {
+        let [key, sep, value] = input.split(/::( |$)/)
+        console.log(key, value)
+        if (key.length === 0) return false;
+        if (!silent) {
+            let token = state.push(id, '', 0)
+            token.meta = {key, value, tag: 'metadata'}
+        }
+        state.pos = key.length + 3;
+        return true
+    }
+    return false
+}
+
+Plugin.prototype.parse = function (state, silent) {
+    return metadataLinkParser(this.id, state, silent)
+}
+
+Plugin.prototype.render = function (tokens, id, options, env) {
+    let {match: link, tag} = tokens[id].meta
+    if (tag === 'metadata') {
+        let {key, value} = tokens[id].meta
+        return '<span class="metadata-key"><a href="/edit/'+key.replace(/ +/g, '_')+'">'+key+'</a></span>: ';
+    }
+}
+
+export default (options) => {
+    return Plugin(options);
+}

editor/src/search.js

@@ -1,27 +1,55 @@
 import $ from 'jquery'
-import qs from 'querystring';
+import qs from 'querystring'
+import * as chrono from 'chrono-node'
+import moment from 'moment';
 
 function search(element) {
     return {
         element: element,
         search(query) {
             element.classList.add('is-loading')
+            let result;
             return startQuery(query)
                 .then(res => {
-                    element.classList.remove('is-loading')
+                    element.classList.remove('is-loading', 'is-error')
+                    result = res
                     return res
+                }).catch(e => {
+                    console.log(e)
+                    element.classList.add('is-error')
+                    return result || []
                 })
         }
     }
 }
 
-function startQuery(query) {
+function startQuery(query, opt) {
+    opt ||= {internal:true}
     return fetch('/search/?' + qs.encode({q: query}))
         .then(res => res.json())
         .then(data => {
             let actualResult = [];
+            if (opt.internal) {
+                actualResult.push({
+                    ref: query.replace(/\s+/g, '_'),
+                    title: 'Create new page "' + query + '"',
+                    line: 'New page',
+                    text: 'New page',
+                })
+                let parseResult = chrono.nl.casual.parse(query)
+                if (parseResult.length) {
+                    let m = moment(parseResult[0].start.date())
+                    actualResult.push({
+                        ref: m.format('LL').replace(/\s+/g, '_'),
+                        title: m.format('LL'),
+                        line: 'Suggested page',
+                        text: 'Suggested page'
+                    })
+                }
+            }
             $.each(data.hits, (key, value) => {
                 actualResult.push({
+                    id: value.id,
                     ref: value.fields.page,
                     title: value.fields.title,
                     line: value.fields.text,

editor/src/sr.js

@@ -0,0 +1,135 @@
+import $ from 'jquery'
+import search from "./search";
+import qs from "querystring";
+
+function fillResult($modal, result) {
+    $modal.data('block-id', result.id)
+    $modal.data('block-original-text', result.line)
+    let visibleText = result.line.replace(/\s*#\S+/g, '').trim();
+    $modal.data('block-text-without-tags', visibleText)
+    $modal.find('.block-title').show().text(result.title)
+    $modal.find('.block-text').show().val(visibleText)
+}
+
+async function replaceBlock(id, oldText) {
+    if (oldText === false) return false;
+
+    await fetch('/api/block/replace', {
+        method: 'post',
+        headers: {
+            'Content-Type': 'application/x-www-form-urlencoded',
+        },
+        body: qs.encode({id, text: oldText}),
+    });
+
+    return true
+}
+
+function appendBlock(id, newText) {
+    return fetch('/api/block/append', {
+        method: 'post',
+        headers: {
+            'Content-Type': 'application/x-www-form-urlencoded',
+        },
+        body: qs.encode({id, text: newText}),
+    });
+}
+
+function resetSRBox(text) {
+    return text + " #review #sr/1"
+}
+
+function processSRBox(review, originalText) {
+    let oldText = originalText
+        .replace(/#sr\/(\d+)/, function (text, srCount) {
+            if (review === 'never') return '';
+            let nextCount = 1;
+            const count = parseInt(srCount)
+            if (review === 'again') nextCount = 1
+            if (review === 'soon') nextCount = count
+            if (review === 'later') nextCount = count + 1
+            return '#sr/' + nextCount;
+        }).trim()
+    if (review === 'never') return oldText.replace(/#review/, '')
+    if (oldText === originalText) return false
+    return oldText
+}
+
+$(function () {
+    let reviewBlocks = [];
+    let isSR = false;
+
+    $('.start-review, .start-sr').on('click', function () {
+        isSR = $(this).hasClass('start-sr')
+
+        // Use different queries for different days
+        const query = isSR ? '+tag:sr tag:"sr/1" tag:"sr/2"' : '+tag:review tag:"sr/1"'
+
+        search.startQuery(query, {internal: false}).then((results) => {
+            reviewBlocks = results
+
+            $('.review-modal .end-of-review').hide();
+            $('.review-modal .review').show();
+
+            let $modal = $('.review-modal').first()
+            $modal.addClass('is-active')
+
+            if (reviewBlocks.length > 0) {
+                const first = reviewBlocks.shift()
+                fillResult($modal, first)
+            } else {
+                $('.review-modal .block-text').hide();
+                $('.review-modal .block-title').hide();
+                $('.review-modal .end-of-review').show();
+                $('.review-modal .review').hide();
+            }
+        })
+
+        return false
+    });
+
+    $('.modal-close, .delete, .close').on('click', function () {
+        $(this).parents('.modal').removeClass('is-active')
+        window.location.reload()
+    });
+
+    $('.modal .show-answer').on('click', function () {
+        $('.review-modal .block-answer').show();
+    });
+
+    $('.modal .review').on('click', function () {
+        const $modal = $(this).parents('.modal')
+        const review = $(this).data('review')
+
+        // NOTE: should we keep the review and sr/* tag in the editable text?
+        const originalText = $modal.data('block-original-text')
+        const originalTextWithoutTags = $modal.data('block-text-without-tags')
+        let text = $modal.find('.block-text').val()
+        let id = $modal.data('block-id')
+
+        processText(review, text, originalTextWithoutTags, originalText, id)
+            .then(() => {
+                if (reviewBlocks.length > 0) {
+                    const first = reviewBlocks.shift()
+                    // reload note with id
+                    search.startQuery('id:' + first.id, {internal: false})
+                        .then((results) => {
+                            fillResult($modal, results[0])
+                        })
+                } else {
+                    $('.review-modal .block-text').hide();
+                    $('.review-modal .block-title').hide();
+                    $('.review-modal .end-of-review').show();
+                    $('.review-modal .review').hide();
+                    reviewBlocks = []
+                }
+            }).catch(e => console.log(e))
+    });
+
+    async function processText(review, text, originalTextWithoutTags, originalText, id) {
+        if (text !== originalTextWithoutTags) {
+            await appendBlock(id, resetSRBox(text))
+        }
+        await replaceBlock(id, processSRBox(review, originalText))
+    }
+});

editor/src/styles.scss

@@ -4,8 +4,13 @@
 @import "~bulma/sass/base/_all";
 @import "~bulma/sass/elements/title.sass";
 @import "~bulma/sass/elements/content.sass";
+@import "~bulma/sass/elements/button.sass";
+@import "~bulma/sass/form/shared.sass";
+@import "~bulma/sass/form/input-textarea.sass";
+@import "~bulma/sass/elements/other.sass";
 @import "~bulma/sass/components/breadcrumb.sass";
 @import "~bulma/sass/components/navbar.sass";
+@import "~bulma/sass/components/modal.sass";
 
 @import '~jquery-contextmenu/dist/jquery.contextMenu.css';
 //@import '~vis-network/styles/vis-network.css';
@@ -29,6 +34,7 @@ body {
     content: "[[";
     color: #ccc;
   }
+
   .content a.wiki-link::after {
     content: "]]";
     color: #ccc;
@@ -78,13 +84,10 @@ body {
   display: none;
 }
 
-.list-item .fold {
-  visibility: hidden;
-}
-
 .list-item:hover .fold {
   visibility: visible;
 }
+
 .list-item.selected .fold {
   visibility: visible;
 }
@@ -94,11 +97,17 @@ body {
 }
 
 .list-item .fold {
+  height: 24px;
+  width: 24px;
+  text-align: center;
   align-self: flex-start;
+  flex-shrink: 0;
   font-size: 10px;
-  margin-top: 4px;
   margin-right: 3px;
   cursor: pointer;
+  visibility: hidden;
+  display: inline-block;
+  padding-top: 4px;
 }
 
 // enable to close multi line content
@@ -194,15 +203,15 @@ mark {
   outline: 4px solid #ffff99;
 }
 
-.root mark::before {
+//.root mark::before {
-  content: "==";
+//  content: "==";
-  color: #999;
+//  color: #999;
-}
+//}
-
+//
-.root mark::after {
+//.root mark::after {
-  content: "==";
+//  content: "==";
-  color: #999;
+//  color: #999;
-}
+//}
 
 .marker, .fold {
   user-select: none;
@@ -222,6 +231,7 @@ mark {
       }
     }
   }
+
   .sidebar-right {
     width: 100%;
     padding: 0 12px;
@@ -242,6 +252,7 @@ mark {
     height: 32px;
     text-align: center;
   }
+
   .calendar-update:active {
     border: 3px solid #aaa;
     border-radius: 4px;
@@ -276,12 +287,15 @@ mark {
       margin-top: 6px;
     }
   }
+
   .week {
     background: #ebebff;
   }
+
   .week a {
     color: black;
   }
+
   .day a {
     position: absolute;
     top: 0;
@@ -290,11 +304,13 @@ mark {
     left: 0;
     color: black;
   }
+
   .day .day-text {
     font-size: 9pt;
     display: block;
     margin-top: 3px;
   }
+
   .day .day-count {
     position: absolute;
     width: 100%;
@@ -306,11 +322,14 @@ mark {
     right: 0;
     bottom: 0;
   }
+
   .today {
     font-weight: bold;
   }
+
   .current {
     background: #f0f0f0;
+
     &.has-content::after {
       font-weight: bold;
       content: '\00B7';
@@ -326,16 +345,19 @@ mark {
 
 .footer {
   padding: 0 12px;
+  margin-top: 300px;
 }
 
 .wiki-list-editor {
   max-width: 960px;
 }
+
 .wiki-list-editor {
   .table-wrapper {
     max-width: 960px;
     overflow-x: auto;
   }
+
   .table-wrapper td {
     white-space: pre;
     text-wrap: none;
@@ -350,6 +372,7 @@ mark {
   flex-grow: 1;
   min-height: 24px;
 }
+
 #autocomplete {
   z-index: 1;
   width: 640px;
@@ -360,25 +383,31 @@ mark {
   background: white;
   border: 1px solid #ccc;
 }
+
 #autocomplete li > a {
   display: block;
 }
+
 #autocomplete li div {
   font-size: 0.8em;
   display: block;
   color: black;
 }
+
 #autocomplete li {
   padding: 4px 16px;
   max-height: 5em;
   overflow: hidden;
 }
+
 #autocomplete li:hover {
   background: #fefefe;
 }
+
 #autocomplete li.selected {
   background: lightblue;
 }
+
 #link-complete {
   z-index: 1;
   width: 280px;
@@ -395,6 +424,7 @@ mark {
   padding: 4px 16px;
   white-space: nowrap;
 }
+
 #link-complete li.selected {
   background: lightblue;
 }
@@ -431,6 +461,7 @@ ins {
 .checklist--item-text {
   align-self: center;
 }
+
 html {
   font-family: 'Inter', sans-serif;
 }
@@ -458,14 +489,26 @@ input.input-line {
   align-items: center;
 }
 
-
 textarea {
   border: none;
   resize: none;
 }
 
 .list-item .content {
-  white-space: pre-wrap;
+  //white-space: pre-wrap;
+}
+
+.wiki-list-editor .content {
+  blockquote {
+    padding-top: 0;
+    padding-bottom: 0;
+  }
+  h1, h2, h3, h4, h5, h6 {
+    margin-bottom: 0;
+  }
+  ul, ol {
+    margin-top: 0;
+  }
 }
 
 .hide {
@@ -475,15 +518,19 @@ textarea {
 .selected {
   background: lightblue;
 }
+
 .fold.closed + .marker {
   border-color: lightblue;
 }
+
 .selected .marker {
   border-color: lightblue;
 }
+
 #editor {
   width: 750px;
 }
+
 .editor.selected .marker {
   /*border-color: white;*/
 }
@@ -540,11 +587,13 @@ input.input-line, input.input-line:active {
 .breadcrumb li {
   max-width: 200px;
 }
+
 .breadcrumb li > a {
   text-overflow: ellipsis;
   overflow: hidden;
   display: block;
 }
+
 .searchbar {
   position: relative;
   display: flex;
@@ -558,14 +607,78 @@ input.input-line, input.input-line:active {
     width: 400px;
   }
 }
+
 .wiki-table .expression {
   display: none;
 }
 
 .todo--todo {
 }
+
 .todo--done {
   text-decoration: line-through;
   text-decoration-skip: leading-spaces;
   color: #999;
 }
+
+
+.tabs {
+  width: 100%;
+}
+
+.tab-bar {
+  display: flex;
+  border-bottom: 1px solid black;
+}
+
+.tab-bar .tab {
+  padding: 8px 16px;
+  cursor: pointer;
+
+  border-bottom: 3px solid white;
+}
+
+.tab-bar .tab:first-child {
+  margin-left: 24px;
+}
+
+.tab-bar .tab.tab-active {
+  border-bottom: 3px solid black;
+}
+
+.tab-bar .tab + .tab {
+  margin-left: 24px;
+}
+
+.tab-page {
+  display: none;
+}
+
+.tab.tab-active {
+}
+
+.tab-page.tab-active {
+  display: block;
+}
+
+.search.input {
+  border: none;
+  padding: 2px;
+}
+
+.search.input.is-error {
+  outline: red solid 4px;
+}
+
+.tag {
+    background: #deeeee;
+    color: #444;
+    border-radius: 3px;
+    padding: 2px 4px;
+}
+.backrefs .tag {
+  background: white;
+}
+
+.review {
+}

editor/src/wikilinks.js

@@ -2,7 +2,26 @@
 
 import Plugin from "markdown-it-regexp";
 import extend from "extend";
-import sanitize from "sanitize-filename";
+
+var illegalRe = /[\/\?<>\\\*\|"]/g;
+var controlRe = /[\x00-\x1f\x80-\x9f]/g;
+var reservedRe = /^\.+$/;
+var windowsReservedRe = /^(con|prn|aux|nul|com[0-9]|lpt[0-9])(\..*)?$/i;
+var windowsTrailingRe = /[\. ]+$/;
+
+function sanitize(input) {
+  const replacement = '';
+  if (typeof input !== 'string') {
+    throw new Error('Input must be string');
+  }
+  var sanitized = input
+    .replace(illegalRe, replacement)
+    .replace(controlRe, replacement)
+    .replace(reservedRe, replacement)
+    .replace(windowsReservedRe, replacement)
+    .replace(windowsTrailingRe, replacement);
+  return sanitized;
+}
 
 export default (options) => {
     const defaults = {
@@ -17,7 +36,7 @@ export default (options) => {
         },
         postProcessPageName: (pageName) => {
             pageName = pageName.trim()
-            pageName = pageName.split('/').map(sanitize).join('/')
+            pageName = pageName.split('/').map(sanitize).map(sanitize).join('/')
             pageName = pageName.replace(/\s+/g, '_')
             return pageName
         },

editor/src/wikilinks2.js

@@ -0,0 +1,82 @@
+'use strict'
+
+var util = require('util')
+
+function Plugin(options) {
+    var self = function (md) {
+        self.options = options
+        self.init(md)
+    }
+
+    self.__proto__ = Plugin.prototype
+    self.id = 'wikilinks'
+
+    return self
+}
+
+util.inherits(Plugin, Function)
+
+Plugin.prototype.init = function (md) {
+    md.inline.ruler.push(this.id, this.parse.bind(this))
+    md.renderer.rules[this.id] = this.render.bind(this)
+}
+
+export function linkParser(id, state, silent) {
+    let input = state.src.slice(state.pos);
+
+    const match = /^#?\[\[/.exec(input)
+    if (!match) {
+        return false
+    }
+
+    let prefixLength = match[0].length
+    let p = state.pos + prefixLength
+    let open = 2
+    while (p < state.src.length - 1 && open > 0) {
+        if (state.src[p] === '[' && state.src[p + 1] === '[') {
+            open += 2
+            p += 2
+        } else if (state.src[p] === ']' && state.src[p + 1] === ']') {
+            open -= 2
+            p += 2
+        } else {
+            p++
+        }
+    }
+
+    if (open === 0) {
+        if (!silent) {
+            let link = state.src.slice(state.pos + prefixLength, p - 2)
+            let token = state.push(id, '', 0)
+            token.meta = {match: link, tag: prefixLength === 3}
+        }
+        state.pos = p
+        return true
+    }
+
+    return false
+}
+
+Plugin.prototype.parse = function (state, silent) {
+    return linkParser(this.id, state, silent)
+}
+
+Plugin.prototype.render = function (tokens, id, options, env) {
+    let {match: link, tag} = tokens[id].meta
+    if (tag) {
+        if (tag === 'metadata') {
+            let {key, value} = tokens[id].meta
+            return '<span class="metadata-key">'+key+'</span>: '+value;
+        }
+        if (link === 'TODO') {
+            return '<input type="checkbox" class="checkbox">';
+        } else if (link === 'DONE') {
+            return '<input type="checkbox" class="checkbox" checked>';
+        }
+    }
+    return '<a href="'+this.options.relativeBaseURL+encodeURIComponent(link.replace(/ +/g, '_')) + '" class="wiki-link">' + (tag ? '#' : '') + link + '</a>';
+}
+
+export default (options) => {
+    return Plugin(options);
+}

editor/test/markdown.test.js

@@ -0,0 +1,70 @@
+/*
+ * Wiki - A wiki with editor
+ * Copyright (c) 2021 Peter Stuifzand
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import assert from 'assert'
+import MarkdownIt from "markdown-it";
+import MarkdownItWikilinks2 from "../src/wikilinks2";
+
+const MD = new MarkdownIt({
+    linkify: false,
+    highlight: function (str, lang) {
+        if (lang === 'mermaid') {
+            return '<div class="mermaid">' + str + '</div>';
+        }
+        return '';
+    }
+})
+
+MD.use(MarkdownItWikilinks2({
+    baseURL: 'http://localhost/',
+    uriSuffix: '',
+    relativeBaseURL: '/edit/',
+    htmlAttributes: {
+        class: 'wiki-link'
+    },
+}))
+
+describe('MD', function () {
+    it('parseLinks', function () {
+        assert.deepStrictEqual(MD.renderInline("#[[TODO]]"), '<input type="checkbox" class="checkbox">')
+        assert.deepStrictEqual(MD.renderInline("#[[TODO]] #[[DONE]]"), '<input type="checkbox" class="checkbox"> <input type="checkbox" class="checkbox" checked>')
+    })
+    it('parseLinks 2', function () {
+        assert.deepStrictEqual(MD.renderInline("#[[TODO]] #[[DONE]]"), '<input type="checkbox" class="checkbox"> <input type="checkbox" class="checkbox" checked>')
+    })
+    it('parseLinks 3', function () {
+        assert.deepStrictEqual(MD.renderInline("test #[[TODO]] test2"), 'test <input type="checkbox" class="checkbox"> test2')
+    })
+    it('parseLinks 4', function () {
+        assert.deepStrictEqual(MD.renderInline("test [[test]] [[test2]] [[test3]]"), 'test <a href="/edit/test" class="wiki-link">test</a> <a href="/edit/test2" class="wiki-link">test2</a> <a href="/edit/test3" class="wiki-link">test3</a>')
+    })
+    it('parseLinks 5', function () {
+        assert.deepStrictEqual(MD.renderInline("test [[test]]"), 'test <a href="/edit/test" class="wiki-link">test</a>')
+    })
+    it('parseLinks 6', function () {
+        assert.deepStrictEqual(MD.renderInline("test [[test]] [[test2]]"), 'test <a href="/edit/test" class="wiki-link">test</a> <a href="/edit/test2" class="wiki-link">test2</a>')
+    })
+
+    it('parseLinks tag', function () {
+        assert.deepStrictEqual(MD.renderInline("test #[[test]]"), 'test <a href="/edit/test" class="wiki-link">#test</a>')
+    })
+
+    it('parseLinks double url', function () {
+        assert.deepStrictEqual(MD.renderInline("[[test [[link]] test2]]"), '<a href="/edit/test_%5B%5Blink%5D%5D_test2" class="wiki-link">test [[link]] test2</a>')
+    })
+})

editor/test/wikilinks2.test.js

@@ -0,0 +1,137 @@
+/*
+ * Wiki - A wiki with editor
+ * Copyright (c) 2021 Peter Stuifzand
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import assert from 'assert'
+import {linkParser} from '../src/wikilinks2'
+
+describe('linkParser', function () {
+    it('parse', function () {
+        let state = {src: '', pos: 0, tokens: []};
+        state.__proto__.push = function (id, s, x) {
+            let token = {id, s, x};
+            this.tokens.push(token)
+            return token
+        }
+        assert.deepStrictEqual(linkParser('test', state), false);
+        assert.deepStrictEqual(state, {
+            src: '',
+            pos: 0,
+            tokens: []
+        })
+    })
+
+    it('parse 2', function () {
+        let state = {src: '[[Link]]', pos: 0, tokens: []};
+        state.__proto__.push = function (id, s, x) {
+            let token = {id, s, x};
+            this.tokens.push(token)
+            return token
+        }
+        assert.deepStrictEqual(linkParser('test', state), true);
+        assert.deepStrictEqual(state, {
+            src: '[[Link]]',
+            pos: 8,
+            tokens: [{
+                id: 'test',
+                s: '',
+                x: 0,
+                meta: {match:'Link', tag: false}
+            }]
+        })
+    })
+
+    it('parse 3', function () {
+        let state = {src: '1234[[Link]]', pos: 4, tokens: []};
+        state.__proto__.push = function (id, s, x) {
+            let token = {id, s, x};
+            this.tokens.push(token)
+            return token
+        }
+        assert.deepStrictEqual(linkParser('test', state), true);
+        assert.deepStrictEqual(state, {
+            src: '1234[[Link]]',
+            pos: 12,
+            tokens: [{
+                id: 'test',
+                s: '',
+                x: 0,
+                meta: {match:'Link', tag: false}
+            }]
+        })
+    })
+
+    it('parse 4', function () {
+        let state = {src: '1234#[[TODO]]', pos: 4, tokens: []};
+        state.__proto__.push = function (id, s, x) {
+            let token = {id, s, x};
+            this.tokens.push(token)
+            return token
+        }
+        assert.deepStrictEqual(linkParser('test', state), true);
+        assert.deepStrictEqual(state, {
+            src: '1234#[[TODO]]',
+            pos: 13,
+            tokens: [{
+                id: 'test',
+                s: '',
+                x: 0,
+                meta: {match:'TODO', tag: true}
+            }]
+        })
+    })
+
+    it('parse text and two links', function () {
+        let state = {src: '1234 [[Link]] [[Link2]]', pos: 5, tokens: []};
+        state.__proto__.push = function (id, s, x) {
+            let token = {id, s, x};
+            this.tokens.push(token)
+            return token
+        }
+        assert.deepStrictEqual(linkParser('test', state), true);
+        assert.deepStrictEqual(state, {
+            src: '1234 [[Link]] [[Link2]]',
+            pos: 13,
+            tokens: [{
+                id: 'test',
+                s: '',
+                x: 0,
+                meta: {match:'Link', tag: false}
+            }]
+        })
+    })
+
+    it('parse text and two links', function () {
+        let state = {src: '1234 [[hello [[world]] Link2]]', pos: 5, tokens: []};
+        state.__proto__.push = function (id, s, x) {
+            let token = {id, s, x};
+            this.tokens.push(token)
+            return token
+        }
+        assert.deepStrictEqual(linkParser('test', state), true);
+        assert.deepStrictEqual(state, {
+            src: '1234 [[hello [[world]] Link2]]',
+            pos: 30,
+            tokens: [{
+                id: 'test',
+                s: '',
+                x: 0,
+                meta: {match:'hello [[world]] Link2', tag: false}
+            }]
+        })
+    })
+})

file.go

@@ -33,7 +33,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/blevesearch/bleve"
+	"github.com/blevesearch/bleve/v2"
 	"github.com/sergi/go-diff/diffmatchpatch"
 )
 
@@ -337,14 +337,19 @@ func (fp *FilePages) save(msg saveMessage) error {
 		if err != nil {
 			return fmt.Errorf("while creating search object %s: %w", page.Name, err)
 		}
+		batch := fp.index.NewBatch()
 		for _, so := range searchObjects {
 			if fp.index != nil {
-				err = fp.index.Index(so.ID, so)
+				err = batch.Index(so.ID, so)
 				if err != nil {
 					return fmt.Errorf("while indexing %s: %w", page.Name, err)
 				}
 			}
 		}
+		err = fp.index.Batch(batch)
+		if err != nil {
+			return fmt.Errorf("while indexing %s: %w", page.Name, err)
+		}
 		sw.Stop()
 		sw.Start("links")
 		err = saveLinksIncremental(fp.dirname, page.Title)
@@ -586,7 +591,7 @@ func saveLinksIncremental(dirname, title string) error {
 	titles[title] = true
 
 	results = nil
-	for t, _ := range titles {
+	for t := range titles {
 		results = append(results, Document{t})
 	}
 
@@ -699,6 +704,9 @@ func (fp *FilePages) PageHistory(p string) ([]Revision, error) {
 		commitId := line[0:start]
 		rest := line[start+1:]
 		pageText := gitRevision(fp.dirname, page, commitId)
+		if pageText == "" {
+			return nil, errors.New("git revision failed")
+		}
 		revisions = append(revisions, Revision{
 			Version: commitId,
 			Page:    DiffPage{Content: pageText},
@@ -727,8 +735,14 @@ func gitRevision(dirname, page, version string) string {
 	cmd.Dir = dirname
 	buf := bytes.Buffer{}
 	cmd.Stdout = &buf
-	cmd.Start()
+	err := cmd.Start()
-	cmd.Wait()
+	if err != nil {
+		return ""
+	}
+	err = cmd.Wait()
+	if err != nil {
+		return ""
+	}
 	return buf.String()
 }
 

go.mod

@@ -3,16 +3,16 @@ module p83.nl/go/wiki
 go 1.14
 
 require (
-	github.com/RoaringBitmap/roaring v0.4.23 // indirect
 	github.com/blevesearch/bleve v1.0.9
+	github.com/blevesearch/bleve/v2 v2.3.0
 	github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d // indirect
 	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
 	github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
-	github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a // indirect
 	github.com/golang/protobuf v1.4.2 // indirect
 	github.com/iancoleman/strcase v0.1.2
 	github.com/jmhodges/levigo v1.0.0 // indirect
@@ -20,12 +20,9 @@ require (
 	github.com/sergi/go-diff v1.1.0
 	github.com/stretchr/testify v1.4.0
 	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect
-	github.com/tinylib/msgp v1.1.2 // indirect
 	github.com/yuin/goldmark v1.1.32
-	go.etcd.io/bbolt v1.3.5 // indirect
 	golang.org/x/net v0.0.0-20200625001655-4c5254603344 // indirect
 	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
-	golang.org/x/text v0.3.3 // indirect
 	google.golang.org/protobuf v1.25.0 // indirect
 	p83.nl/go/ekster v0.0.0-20191119211024-4511657daa0b
 	p83.nl/go/indieauth v0.1.0

go.sum

@@ -2,22 +2,35 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
 github.com/RoaringBitmap/roaring v0.4.21/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo=
-github.com/RoaringBitmap/roaring v0.4.23 h1:gpyfd12QohbqhFO4NVDUdoPOCXsyahYRQhINmlHxKeo=
+github.com/RoaringBitmap/roaring v0.9.4 h1:ckvZSX5gwCRaJYBNe7syNawCU5oruY9gQmjXlp4riwo=
-github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo=
+github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA=
 github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA=
+github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
 github.com/blevesearch/bleve v1.0.9 h1:kqw/Ank/61UV9/Bx9kCcnfH6qWPgmS8O5LNfpsgzASg=
 github.com/blevesearch/bleve v1.0.9/go.mod h1:tb04/rbU29clbtNgorgFd8XdJea4x3ybYaOjWKr+UBU=
+github.com/blevesearch/bleve/v2 v2.3.0 h1:5XKlSdpcjeJdE7n0FUEDeJRJwLuhPxq+k5n7h5UaJkg=
+github.com/blevesearch/bleve/v2 v2.3.0/go.mod h1:egW/6gZEhM3oBvRjuHXGvGb92cKZ9867OqPZAmCG8MQ=
+github.com/blevesearch/bleve_index_api v1.0.1 h1:nx9++0hnyiGOHJwQQYfsUGzpRdEVE5LsylmmngQvaFk=
+github.com/blevesearch/bleve_index_api v1.0.1/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4=
 github.com/blevesearch/blevex v0.0.0-20190916190636-152f0fe5c040 h1:SjYVcfJVZoCfBlg+fkaq2eoZHTf5HaJfaTeTkOtyfHQ=
 github.com/blevesearch/blevex v0.0.0-20190916190636-152f0fe5c040/go.mod h1:WH+MU2F4T0VmSdaPX+Wu5GYoZBrYWdOZWSjzvYcDmqQ=
 github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
 github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M=
-github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0=
 github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA=
+github.com/blevesearch/mmap-go v1.0.3 h1:7QkALgFNooSq3a46AE+pWeKASAZc9SiNFJhDGF1NDx4=
+github.com/blevesearch/mmap-go v1.0.3/go.mod h1:pYvKl/grLQrBxuaRYgoTssa4rVujYYeenDp++2E+yvs=
+github.com/blevesearch/scorch_segment_api/v2 v2.1.0 h1:NFwteOpZEvJk5Vg0H6gD0hxupsG3JYocE4DBvsA2GZI=
+github.com/blevesearch/scorch_segment_api/v2 v2.1.0/go.mod h1:uch7xyyO/Alxkuxa+CGs79vw0QY8BENSBjg6Mw5L5DE=
 github.com/blevesearch/segment v0.9.0 h1:5lG7yBCx98or7gK2cHMKPukPZ/31Kag7nONpoBt22Ac=
 github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ=
 github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s=
 github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs=
+github.com/blevesearch/upsidedown_store_api v1.0.1 h1:1SYRwyoFLwG3sj0ed89RLtM15amfX2pXlYbFOnF8zNU=
+github.com/blevesearch/upsidedown_store_api v1.0.1/go.mod h1:MQDVGpHZrpe3Uy26zJBf/a8h0FZY6xJbthIMm8myH2Q=
+github.com/blevesearch/vellum v1.0.7 h1:+vn8rfyCRHxKVRgDLeR0FAXej2+6mEb5Q15aQE/XESQ=
+github.com/blevesearch/vellum v1.0.7/go.mod h1:doBZpmRhwTsASB4QdUZANlJvqVAUdUyX0ZK7QJCTeBE=
 github.com/blevesearch/zap/v11 v11.0.9 h1:wlSrDBeGN1G4M51NQHIXca23ttwUfQpWaK7uhO5lRSo=
 github.com/blevesearch/zap/v11 v11.0.9/go.mod h1:47hzinvmY2EvvJruzsSCJpro7so8L1neseaGjrtXHOY=
 github.com/blevesearch/zap/v12 v12.0.9 h1:PpatkY+BLVFZf0Ok3/fwgI/I4RU0z5blXFGuQANmqXk=
@@ -26,6 +39,16 @@ github.com/blevesearch/zap/v13 v13.0.1 h1:NSCM6uKu77Vn/x9nlPp4pE1o/bftqcOWZEHSyZ
 github.com/blevesearch/zap/v13 v13.0.1/go.mod h1:XmyNLMvMf8Z5FjLANXwUeDW3e1+o77TTGUWrth7T9WI=
 github.com/blevesearch/zap/v14 v14.0.0 h1:HF8Ysjm13qxB0jTGaKLlatNXmJbQD8bY+PrPxm5v4hE=
 github.com/blevesearch/zap/v14 v14.0.0/go.mod h1:sUc/gPGJlFbSQ2ZUh/wGRYwkKx+Dg/5p+dd+eq6QMXk=
+github.com/blevesearch/zapx/v11 v11.3.2 h1:TDdcbaA0Yz3Y5zpTrpvyW1AeicqWTJL3g8D5g48RiHM=
+github.com/blevesearch/zapx/v11 v11.3.2/go.mod h1:YzTfUm4kS3e8OmTXDHVV8OzC5MWPO/VPJZQgPNVb4Lc=
+github.com/blevesearch/zapx/v12 v12.3.2 h1:XB09XMg/3ibeIJRCm2zjkaVwrtAuk6c55YRSmVlwUDk=
+github.com/blevesearch/zapx/v12 v12.3.2/go.mod h1:RMl6lOZqF+sTxKvhQDJ5yK2LT3Mu7E2p/jGdjAaiRxs=
+github.com/blevesearch/zapx/v13 v13.3.2 h1:mTvALh6oayreac07VRAv94FLvTHeSBM9sZ1gmVt0N2k=
+github.com/blevesearch/zapx/v13 v13.3.2/go.mod h1:eppobNM35U4C22yDvTuxV9xPqo10pwfP/jugL4INWG4=
+github.com/blevesearch/zapx/v14 v14.3.2 h1:oW36JVaZDzrzmBa1X5jdTIYzdhkOQnr/ie13Cb2X7MQ=
+github.com/blevesearch/zapx/v14 v14.3.2/go.mod h1:zXNcVzukh0AvG57oUtT1T0ndi09H0kELNaNmekEy0jw=
+github.com/blevesearch/zapx/v15 v15.3.2 h1:OZNE4CQ9hQhnB21ySC7x2/9Q35U3WtRXLAh5L2gdCXc=
+github.com/blevesearch/zapx/v15 v15.3.2/go.mod h1:C+f/97ZzTzK6vt/7sVlZdzZxKu+5+j4SrGCvr9dJzaY=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -33,6 +56,7 @@ github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8Nz
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k=
 github.com/couchbase/moss v0.1.0/go.mod h1:9MaHIaRuy9pvLPUJxB8sh8OrLfyDczECVL37grCIubs=
+github.com/couchbase/moss v0.2.0/go.mod h1:9MaHIaRuy9pvLPUJxB8sh8OrLfyDczECVL37grCIubs=
 github.com/couchbase/vellum v1.0.1 h1:qrj9ohvZedvc51S5KzPfJ6P6z0Vqzv7Lx7k3mVc2WOk=
 github.com/couchbase/vellum v1.0.1/go.mod h1:FcwrEivFpNi24R3jLOs3n+fs5RnuQnQqCLBJ1uAg1W4=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
@@ -57,9 +81,6 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
-github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a h1:FQqoVvjbiUioBBFUL5up+h+GdCa/AnJsL/1bIs/veSI=
-github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
-github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8=
 github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -82,7 +103,6 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 h1:twflg0XRTjwKpxb/jFExr4HGq6on2dEOmnL6FV+fgPw=
 github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
@@ -92,7 +112,6 @@ github.com/iancoleman/strcase v0.1.2/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5N
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U=
 github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/kljensen/snowball v0.6.0/go.mod h1:27N7E8fVU5H68RlUmnWwZCfxgt4POBJfENGMvNRhldw=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -114,7 +133,6 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
 github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -142,8 +160,6 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP
 github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok=
 github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8=
 github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
-github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ=
-github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc=
 github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
@@ -188,8 +204,8 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=

link/parser.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"html/template"
 	"io"
+	"log"
 	"strings"
 )
 
@@ -76,6 +77,24 @@ func formatTitle(w io.Writer, input string, root Tree, indent int) {
 	}
 }
 
+func findAllLinks(input string, root Tree) []string {
+	var links []string
+	typ := root.cur.typ
+	if typ == "link" {
+		links = append(links, root.children[1].text(input))
+	}
+	for _, c := range root.children {
+		links = append(links, findAllLinks(input, c)...)
+	}
+	return links
+}
+
+func FindAllLinks(input string) []string {
+	var p Parser
+	root := p.Parse(input)
+	return findAllLinks(input, root)
+}
+
 func FormatHtmlTitle(input string) template.HTML {
 	p := Parser{}
 	root := p.Parse(input)
@@ -87,13 +106,16 @@ func FormatHtmlTitle(input string) template.HTML {
 func (p *Parser) Parse(input string) Tree {
 	p.stack = append(p.stack, Tree{})
 
+	limit := 1000
+
 	i := 0
 
 	p.pushMarker(i)
-	for i < len(input) {
+	for i < len(input) && limit > 0 {
 		p.pushMarker(i)
 		for i < len(input) && (input[i] != '[' && input[i] != ']') {
 			i++
+			limit--
 		}
 		p.popMarker(i, "text")
 		if i+2 <= len(input) && input[i:i+2] == "[[" {
@@ -109,8 +131,13 @@ func (p *Parser) Parse(input string) Tree {
 			p.popMarker(i, "end link tag")
 			p.popMarker(i, "link")
 		}
+		limit--
 	}
 	p.popMarker(i, "full text")
 
+	if limit == 0 {
+		log.Println("LIMIT REACHED: ", input)
+	}
+
 	return p.output()
 }

link/parser_test.go

@@ -25,15 +25,31 @@ import (
 )
 
 func TestFormatHtmlTitle(t *testing.T) {
-
 	tests := []struct {
 		input, output string
 	}{
 		{input: "hello", output: "hello"},
 		{input: "hello [[world]]", output: `hello <a href="world">[[world]]</a>`},
+		{input: "hello [[world]] end", output: `hello <a href="world">[[world]]</a> end`},
+		{input: "hello [[world [[current stuff]] here]] end", output: `hello <a href="world [[current stuff]] here">[[world [[current stuff]] here]]</a> end`},
 	}
 	for _, test := range tests {
 		s := FormatHtmlTitle(test.input)
 		assert.Equal(t, test.output, string(s))
 	}
 }
+func TestFindAllLinks(t *testing.T) {
+	tests := []struct {
+		input  string
+		output []string
+	}{
+		{input: "hello", output: nil},
+		{input: "hello [[world]]", output: []string{"world"}},
+		{input: "hello [[world]] end", output: []string{"world"}},
+		{input: "hello [[world [[current stuff]] here]] end", output: []string{"world [[current stuff]] here", "current stuff"}},
+	}
+	for _, test := range tests {
+		links := FindAllLinks(test.input)
+		assert.Equal(t, test.output, links)
+	}
+}

list-editor/index.js

@@ -6,7 +6,8 @@ import textareaAutosizeInit from "./textarea.autosize"
 import createCursor from './cursor'
 import Store from './store'
 import Keymap from './keymap'
-import getCaretCoordinates from "../editor/src/caret-position";
+import getCaretCoordinates from "../editor/src/caret-position"
+import TurndownService from "turndown"
 
 textareaAutosizeInit($)
 
@@ -73,8 +74,10 @@ function editor(root, inputData, options) {
 
     function save() {
         return new Promise(function (resolve, reject) {
+
             if (store.hasChanged()) {
-                resolve(store.debug().result)
+                let result = store.debug().result
+                resolve(result)
                 store.clearChanged()
             }
         });
@@ -109,12 +112,13 @@ function editor(root, inputData, options) {
         });
     }
 
-    function flat(element) {
+    function flat(element, opt) {
+        opt = opt || {}
         let item = $(element).parents('.list-item')
         let id = item.attr('data-id')
 
         return new Promise(function (resolve, reject) {
-            resolve(store.flat(id));
+            resolve(store.flat(id, opt));
         });
     }
 
@@ -233,6 +237,10 @@ function editor(root, inputData, options) {
         return {indented: indented, text: '', fold: 'open', hidden: false}
     }
 
+    function hasMarkdown(text) {
+        return text.match(/[^a-zA-Z0-9 .,!@$&"'?]/)
+    }
+
     function newItem(value) {
         let el = document.createElement('div')
         el.classList.add('list-item')
@@ -249,7 +257,11 @@ function editor(root, inputData, options) {
 
         line.prepend(content)
 
+        if (hasMarkdown(value.text)) {
             options.transform(value.text, $(content), value.id, EDITOR)
+        } else {
+            $(content).html(value.text)
+        }
 
         let marker = document.createElement('span')
         marker.classList.add('marker')
@@ -318,9 +330,12 @@ function editor(root, inputData, options) {
 
         let hideLevel = 99999;
 
+        let closedFolds = JSON.parse(localStorage.getItem('closed-folds') || '{}') || {}
+
         $enter.each(function (index, li) {
             let storeId = enterData[index]
             let value = rootData.value(storeId)
+            value.fold = closedFolds[value.id] ? 'closed' : 'open'
 
             let hasChildren = false;
             if (index + 1 < last) {
@@ -336,7 +351,12 @@ function editor(root, inputData, options) {
 
             value.hidden = value.indented >= hideLevel
 
-            options.transform(value.text, $li.find('.content'), value.id, EDITOR)
+            let $content = $li.find('.content');
+            if (hasMarkdown(value.text)) {
+                options.transform(value.text, $content, value.id, EDITOR)
+            } else {
+                $content.html(value.text)
+            }
 
             if (value.indented < hideLevel) {
                 if (value.fold !== 'open') {
@@ -356,6 +376,8 @@ function editor(root, inputData, options) {
 
         _.each(exitData, function (storeId, index) {
             let value = rootData.value(storeId)
+            value.fold = closedFolds[value.id] ? 'closed' : 'open'
+
             let $li = newItem(value)
                 .css('margin-left', (value.indented * 32) + 'px')
                 .toggleClass('selected', cursor.atPosition(index + $enter.length))
@@ -396,15 +418,19 @@ function editor(root, inputData, options) {
     }
 
     function enableDragging(rootElement) {
+        let start = -1;
+        let startID = null;
+
         let drake = dragula([rootElement], {
             moves: function (el, container, handle, sibling) {
                 return handle.classList.contains('marker')
+            },
+            accepts: function (el, target, source, sibling) {
+                el.style.marginLeft = sibling === null ? 0 : sibling.style.marginLeft
+                return true
             }
         })
 
-        let start = -1;
-        let startID = null;
-
         drake.on('drag', function (el, source) {
             startID = $(el).attr('data-id')
         })
@@ -421,6 +447,7 @@ function editor(root, inputData, options) {
 
             let newPosition = store.moveBefore(startID, stopID)
             cursor.set(newPosition[0])
+            // fix indent
 
             _.defer(() => {
                 trigger('change')
@@ -495,9 +522,37 @@ function editor(root, inputData, options) {
             return true
         }
 
-        let pastedData = event.originalEvent.clipboardData.getData('text/plain')
+        if (event.originalEvent.clipboardData.types.includes("text/html")) {
-        let items = JSON.parse(pastedData.toString());
+            let pastedData = event.originalEvent.clipboardData.getData('text/html')
+            const turndownService = new TurndownService({
+                headingStyle: 'atx',
+                codeBlockStyle: 'fenced',
+            })
+            const markdown = turndownService.turndown(pastedData)
+            let items = markdown.split(/\n+/);
+            console.log(items)
+            let item = $(this).parents('.list-item')
+            let id = item.attr('data-id')
+            const firstItem = store.value(id)
+            items = _.map(items, text => {
+                const m = text.match(/^(\s*)\*\s*(.*)$/)
+                if (m) {
+                    const item = newListItem(firstItem.indented+1+Math.trunc(m[1].length/4))
+                    item.text = m[2]
+                    return item
+                }
+                const item = newListItem(firstItem.indented)
+                item.text = text
+                return item
+            })
+            store.insertAfter(id, ...items)
+            trigger('change')
+            return false
+        }
 
+        let pastedData = event.originalEvent.clipboardData.getData('text/plain')
+        try {
+            let items = JSON.parse(pastedData.toString());
             let item = $(this).parents('.list-item')
             let id = item.attr('data-id')
 
@@ -507,10 +562,46 @@ function editor(root, inputData, options) {
             })
 
             store.insertAfter(id, ...items)
-
             trigger('change')
-
             return false
+        } catch (e) {
+            let inputString = pastedData.toString()
+            if (inputString.match(/^- /)) {
+                let item = $(this).parents('.list-item')
+                let id = item.attr('data-id')
+                let lines = inputString.split(/^( *)- /ms)
+                lines.shift()
+                const firstItem = store.value(id)
+                const firstIndent = firstItem.indented
+
+                let items = _.map(_.chunk(lines, 2), line => {
+                    const indent = Math.trunc(line[0].length / 4);
+                    const item = newListItem(firstItem.indented+indent)
+                    item.text = _.trimEnd(line[1]).replace(/\n/g, "  \n")
+                    return item
+                })
+                store.insertAfter(id, ...items)
+                trigger('change')
+                return false
+            } else {
+                let items = inputString.split(/\n+/);
+                let item = $(this).parents('.list-item')
+                let id = item.attr('data-id')
+                if (items.length === 1) {
+                    return true
+                } else {
+                    const firstItem = store.value(id)
+                    items = _.map(items, text => {
+                        const item = newListItem(firstItem.indented)
+                        item.text = text
+                        return item
+                    })
+                    store.insertAfter(id, ...items)
+                    trigger('change')
+                }
+                return false
+            }
+        }
     });
 
     function moveCursor(event, dir) {

list-editor/package-lock.json

@@ -53,6 +53,11 @@
       "resolved": "https://registry.npmjs.org/custom-event/-/custom-event-1.0.0.tgz",
       "integrity": "sha1-LkYovhncSyFLXAJjDFlx6BFhgGI="
     },
+    "domino": {
+      "version": "2.1.6",
+      "resolved": "https://registry.npmjs.org/domino/-/domino-2.1.6.tgz",
+      "integrity": "sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ=="
+    },
     "dragula": {
       "version": "3.7.2",
       "resolved": "https://registry.npmjs.org/dragula/-/dragula-3.7.2.tgz",
@@ -164,6 +169,14 @@
       "resolved": "https://registry.npmjs.org/ticky/-/ticky-1.0.1.tgz",
       "integrity": "sha1-t8+nHnaPHJAAxJe5FRswlHxQ5G0="
     },
+    "turndown": {
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.1.1.tgz",
+      "integrity": "sha512-BEkXaWH7Wh7e9bd2QumhfAXk5g34+6QUmmWx+0q6ThaVOLuLUqsnkq35HQ5SBHSaxjSfSM7US5o4lhJNH7B9MA==",
+      "requires": {
+        "domino": "^2.1.6"
+      }
+    },
     "wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",

list-editor/package.json

@@ -4,7 +4,9 @@
   "description": "Simple editor of lists",
   "author": "Peter Stuifzand <peter@p83.nl>",
   "main": "index.js",
-  "sideEffects": ["**"],
+  "sideEffects": [
+    "**"
+  ],
   "scripts": {
     "test": "jasmine --require=esm"
   },
@@ -14,7 +16,8 @@
     "dragula": "^3.7.2",
     "he": "^1.2.0",
     "jquery": "^3.5.1",
-    "lodash": "^4.17.19"
+    "lodash": "^4.17.19",
+    "turndown": "^7.1.1"
   },
   "devDependencies": {
     "esm": "^3.2.25",

list-editor/spec/cursor.spec.js

@@ -47,4 +47,58 @@ describe("A cursor", function() {
             expect(this.cursor.get()).toBe(3)
         })
     })
+
+    describe("with a store with current closed", function () {
+        beforeEach(function () {
+            this.store = createStore([
+                {indented:0, fold: 'open'},
+                {indented:1, fold: 'open'},
+                {indented:2, fold: 'open'},
+                {indented:3, fold: 'open'},
+                {indented:1, fold: 'closed', hidden: true},
+            ])
+        })
+
+        it("moveUp moves up by one", function() {
+            this.cursor.set(4)
+            this.cursor.moveUp(this.store)
+            expect(this.cursor.get()).toBe(3)
+        })
+    })
+
+    describe("with a store with above closed", function () {
+        beforeEach(function () {
+            this.store = createStore([
+                {indented:0, fold: 'open'},
+                {indented:1, fold: 'open'},
+                {indented:2, fold: 'open'},
+                {indented:3, fold: 'closed', hidden: true},
+                {indented:1, fold: 'open'},
+            ])
+        })
+
+        it("moveUp moves up by one", function() {
+            this.cursor.set(4)
+            this.cursor.moveUp(this.store)
+            expect(this.cursor.get()).toBe(2)
+        })
+    })
+
+    describe("with a store with multiple above closed", function () {
+        beforeEach(function () {
+            this.store = createStore([
+                {indented:0, fold: 'open'},
+                {indented:1, fold: 'open'},
+                {indented:2, fold: 'closed', hidden: true},
+                {indented:3, fold: 'closed', hidden: true},
+                {indented:1, fold: 'open'},
+            ])
+        })
+
+        it("moveUp moves up by one", function() {
+            this.cursor.set(4)
+            this.cursor.moveUp(this.store)
+            expect(this.cursor.get()).toBe(1)
+        })
+    })
 })

list-editor/spec/store.spec.js

@@ -189,4 +189,40 @@ describe("A store", function () {
             expect(store.debug().idList).toEqual(["_a", "_c", "_b", "_d"])
         })
     })
+
+    describe("contains a moveBefore method which handles indentation", function () {
+        let store
+
+        beforeEach(function () {
+            store = createStore([
+                {text: "a", id: "_a", indented: 0},
+                {text: "b", id: "_b", indented: 1},
+                {text: "c", id: "_c", indented: 2},
+                {text: "d", id: "_d", indented: 3},
+                {text: "e", id: "_e", indented: 0},
+            ])
+        })
+        it("moveBefore _e, _c", function () {
+            store.moveBefore("_e", "_c")
+            let debug = store.debug();
+            expect(debug.idList).toEqual(["_a", "_b", "_e", "_c", "_d"])
+            expect(debug.result[2]).toEqual({text: "e", id: "_e", indented: 2})
+        })
+        it("moveBefore _d, at-end", function () {
+            store.moveBefore("_d", "at-end")
+            let debug = store.debug();
+            expect(debug.idList).toEqual(["_a", "_b", "_c", "_e", "_d"])
+            expect(debug.result[4]).toEqual({text: "d", id: "_d", indented: 0})
+        })
+        it("moveBefore _b, at-end", function () {
+            store.moveBefore("_b", "at-end")
+            let debug = store.debug();
+            expect(debug.idList).toEqual(["_a", "_e", "_b", "_c", "_d"])
+            expect(debug.result[0]).toEqual({text: "a", id: "_a", indented: 0})
+            expect(debug.result[1]).toEqual({text: "e", id: "_e", indented: 0})
+            expect(debug.result[2]).toEqual({text: "b", id: "_b", indented: 0})
+            expect(debug.result[3]).toEqual({text: "c", id: "_c", indented: 1})
+            expect(debug.result[4]).toEqual({text: "d", id: "_d", indented: 2})
+        })
+    })
 })

list-editor/store.js

@@ -55,21 +55,15 @@ function Store(inputData) {
     }
 
     function prevCursorPosition(cursor) {
-        let curIndent = values[idList[cursor]].indented
-        let curClosed = values[idList[cursor]].fold !== 'open';
-        if (!curClosed) {
-            curIndent = 10000000;
-        }
         let moving = true
 
         while (moving) {
             cursor--
             if (cursor < 0) {
                 cursor = idList.length - 1
-                curIndent = values[idList[cursor]].indented
             }
             let next = values[idList[cursor]];
-            if (curIndent >= next.indented && !next.hidden) {
+            if (!next.hidden) {
                 moving = false
             }
         }
@@ -203,9 +197,20 @@ function Store(inputData) {
             values[currentId] = newValue
             changed = true
         }
+        updateFold(currentId, newValue.fold === 'open')
         return currentId
     }
 
+    function updateFold(id, open) {
+        let closedFolds = JSON.parse(localStorage.getItem("closed-folds") || '{}') || {}
+        if (open) {
+            delete closedFolds[id]
+        } else {
+            closedFolds[id] = true
+        }
+        localStorage.setItem('closed-folds', JSON.stringify(closedFolds))
+    }
+
     function length() {
         return idList.length;
     }
@@ -301,7 +306,19 @@ function Store(inputData) {
             rotate(idList, toIndex, fromIndex + n, fromIndex - toIndex)
             changed = true
         }
-        return [index(from), n]
+
+        // Copy indent from the next item, or 0 when at the end
+        const v = value(to)
+        const diff = (v ? v.indented : 0) - value(from).indented
+
+        let first = index(from)
+        let i = 0
+        while (i < n) {
+            value(idList[first + i]).indented += diff
+            i++
+        }
+
+        return [first, n]
     }
 
     /**
@@ -342,8 +359,23 @@ function Store(inputData) {
         return [values[from], ..._.takeWhile(items, item => item.indented > values[from].indented)]
     }
 
-    function flat(from) {
+    function flat(from, opt) {
-        return selectItemsFrom(from)
+        opt = opt || {}
+        let result = selectItemsFrom(from)
+        if (opt.base && result.length > 0) {
+            const first = result[0]
+            let children = _.map(result.slice(1), (item) => {
+                let newItem = _.clone(item)
+                newItem.indented -= first.indented+1
+                newItem.id = ID()
+                return newItem
+            })
+            return {
+                title: first.text,
+                children
+            }
+        }
+        return result
     }
 
     /**
@@ -455,7 +487,7 @@ function Store(inputData) {
 
     let removeLevel = 9999999999;
     _.each(inputData, (d) => {
-        if (d.text.startsWith("{{query:")) {
+        if (d.text && d.text.startsWith("{{query:")) {
             removeLevel = d.indented;
             append(d)
         } else if (d.indented <= removeLevel) {

main.go

@@ -28,6 +28,7 @@ import (
 	"html/template"
 	"io"
 	"log"
+	"math/rand"
 	"net/http"
 	"net/url"
 	"os"
@@ -36,14 +37,16 @@ import (
 	"strings"
 	"time"
 
-	"github.com/blevesearch/bleve"
 	"p83.nl/go/ekster/pkg/util"
 	"p83.nl/go/indieauth"
 	"p83.nl/go/wiki/link"
+
+	"github.com/blevesearch/bleve/v2"
 )
 
 func init() {
 	log.SetFlags(log.Lshortfile)
+	rand.Seed(time.Now().UnixNano())
 }
 
 type authorizedKey string
@@ -229,7 +232,7 @@ func (*authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	if r.Method == http.MethodGet {
 		if r.URL.Path == "/auth/login" {
-			templates := baseTemplate
+			templates := []string{"templates/layout_no_sidebar.html"}
 			templates = append(templates, "templates/login.html")
 			t, err := template.ParseFiles(templates...)
 			if err != nil {
@@ -288,10 +291,14 @@ func (*authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			urlString := r.PostFormValue("url")
 
 			u, err := url.Parse(urlString)
+			if err != nil {
+				urlString = fmt.Sprintf("https://%s/", urlString)
+				u, err = url.Parse(urlString)
 				if err != nil {
 					http.Error(w, err.Error(), 400)
 					return
 				}
+			}
 
 			endpoints, err := indieauth.GetEndpoints(u)
 			if err != nil {
@@ -738,7 +745,7 @@ func (h *graphHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 func (h *indexHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	defer r.Body.Close()
 
-	metaKV, err := regexp.Compile(`(\w+)::\s+(.*)`)
+	metaKV, err := regexp.Compile(`(\w[ \w]*)::(?: +(.*))?`)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -839,7 +846,7 @@ func (h *indexHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 					for _, line := range lines {
 						if first {
 							builder.WriteString(strings.Repeat("  ", item.Indented))
-							builder.WriteString("* ")
+							builder.WriteString("- ")
 							builder.WriteString(line)
 							builder.WriteByte('\n')
 							first = false
@@ -856,7 +863,7 @@ func (h *indexHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 						}
 					}
 					builder.WriteString(strings.Repeat("  ", item.Indented))
-					builder.WriteString("* ")
+					builder.WriteString("- ")
 					builder.WriteString(item.Text)
 					builder.WriteByte('\n')
 				}
@@ -872,7 +879,7 @@ func (h *indexHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if format == "html" {
-		pageText = metaKV.ReplaceAllString(pageText, "**[[$1]]**: $2")
+		pageText = metaKV.ReplaceAllString(pageText, "**$1**: $2")
 
 		pageText = renderLinks(pageText, false)
 		pageText = renderMarkdown2(pageText)
@@ -892,7 +899,7 @@ func (h *indexHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			ShowGraph:    page != "Daily_Notes",
 			TodayPage:    "Today",
 		}
-		templates := baseTemplate
+		templates := []string{"templates/layout_no_sidebar.html"}
 		templates = append(templates, "templates/view.html")
 		t, err := template.ParseFiles(templates...)
 		if err != nil {
@@ -918,11 +925,7 @@ func (h *indexHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 }
 
 func renderLinks(pageText string, edit bool) string {
-	hrefRE, err := regexp.Compile(`#?\[\[\s*([^\]]+)\s*\]\]`)
+	hrefRE := regexp.MustCompile(`#?\[\[\s*([^\]]+)\s*\]\]`)
-	if err != nil {
-		log.Fatal(err)
-	}
-
 	pageText = hrefRE.ReplaceAllStringFunc(pageText, func(s string) string {
 		tag := false
 		if s[0] == '#' {
@@ -934,8 +937,16 @@ func renderLinks(pageText string, edit bool) string {
 		s = strings.TrimSuffix(s, "]]")
 		s = strings.TrimSpace(s)
 		if tag {
+			switch s {
+			case "TODO":
+				return fmt.Sprint(`[ ] `)
+			case "DONE":
+				return fmt.Sprint(`[X] `)
+			default:
 				return fmt.Sprintf(`<a href=%q class="tag">%s</a>`, cleanNameURL(s), s)
 			}
+		}
+
 		editPart := ""
 		if edit {
 			editPart = "edit/"
@@ -943,6 +954,14 @@ func renderLinks(pageText string, edit bool) string {
 
 		return fmt.Sprintf("[%s](/%s%s)", s, editPart, cleanNameURL(s))
 	})
+
+	tagRE := regexp.MustCompile(`#(\S+)`)
+	pageText = tagRE.ReplaceAllStringFunc(pageText, func(s string) string {
+		s = strings.TrimPrefix(s, "#")
+		s = strings.TrimSpace(s)
+		return fmt.Sprintf(`<a href="/%s" class="tag">%s</a>`, url.PathEscape(cleanNameURL(s)), s)
+	})
+
 	return pageText
 }
 
@@ -1044,8 +1063,39 @@ func main() {
 	}
 
 	mp = NewFilePages(dataDir, searchIndex)
+	repo := NewBlockRepo("data")
 
 	http.Handle("/auth/", &authHandler{})
+	http.HandleFunc("/api/block/view", wrapAuth(func(w http.ResponseWriter, r *http.Request) {
+		defer r.Body.Close()
+
+		if !r.Context().Value(authKey).(bool) {
+			http.Error(w, "Unauthorized", 401)
+			return
+		}
+
+		if r.Method != "GET" {
+			http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
+			return
+		}
+
+		id := r.URL.Query().Get("id")
+
+		block, err := repo.Load(id)
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+			return
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+		w.Header().Set("Cache-Control", "no-store")
+		enc := json.NewEncoder(w)
+		enc.SetIndent("", "  ")
+		err = enc.Encode(block)
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+		}
+	}))
 	http.HandleFunc("/api/block/", wrapAuth(func(w http.ResponseWriter, r *http.Request) {
 		defer r.Body.Close()
 
@@ -1103,6 +1153,36 @@ func main() {
 			}
 		}
 	}))
+	http.HandleFunc("/api/block/replace", wrapAuth(func(w http.ResponseWriter, r *http.Request) {
+		err := r.ParseForm()
+		if err != nil {
+			http.Error(w, err.Error(), 400)
+			return
+		}
+		id := r.Form.Get("id")
+		if id == "" {
+			http.Error(w, "missing id", 400)
+			return
+		}
+		block, err := repo.Load(id)
+		block.Text = r.Form.Get("text")
+		err = repo.Save(id, block)
+		// update search index
+		searchObjects, err := createSearchObjects(id)
+		batch := searchIndex.NewBatch()
+		for _, so := range searchObjects {
+			err = batch.Index(so.ID, so)
+			if err != nil {
+				log.Println(err)
+			}
+		}
+		err = searchIndex.Batch(batch)
+		if err != nil {
+			log.Println(err)
+		}
+		// TODO: update backlinks
+		return
+	}))
 	http.HandleFunc("/api/block/append", wrapAuth(func(w http.ResponseWriter, r *http.Request) {
 		err := r.ParseForm()
 		if err != nil {
@@ -1114,46 +1194,38 @@ func main() {
 			http.Error(w, "missing id", 400)
 			return
 		}
-
+		newBlock := Block{
-		page := mp.Get(id)
+			Text:     r.Form.Get("text"),
-		log.Println(page.Content)
+			Children: []string{},
-		var listItems []ListItem
+			Parent:   id,
-		id = page.Name // Use the name that was actually loaded
-
-		err = json.NewDecoder(strings.NewReader(page.Content)).Decode(&listItems)
-		if err != nil && err != io.EOF {
-			http.Error(w, fmt.Sprintf("while decoding: %s", err.Error()), 500)
-			return
 		}
-
 		newId := &ID{"1", true}
 		generatedID := newId.NewID()
-		listItems = append(listItems, ListItem{
+		err = repo.Save(generatedID, newBlock)
-			ID:       generatedID,
+		block, err := repo.Load(id)
-			Indented: 0,
+		block.Children = append(block.Children, generatedID)
-			Text:     r.Form.Get("text"),
+		err = repo.Save(id, block)
-			Fleeting: false,
-		})
 
-		var buf bytes.Buffer
+		// update search index
+		sw := stopwatch{}
+		sw.Start("createSearchObjects")
+		searchObjects, err := createSearchObjects(id)
 
-		err = json.NewEncoder(&buf).Encode(&listItems)
+		batch := searchIndex.NewBatch()
+
+		for _, so := range searchObjects {
+			err = batch.Index(so.ID, so)
 			if err != nil {
-			http.Error(w, fmt.Sprintf("while encoding: %s", err.Error()), 500)
+				log.Println(err)
-			return
+			}
 		}
 
-		page.Content = buf.String()
+		err = searchIndex.Batch(batch)
-		page.Name = id
-		page.Title = id
-
-		err = mp.Save(id, page, "", "")
 		if err != nil {
-			http.Error(w, fmt.Sprintf("while saving: %s", err.Error()), 500)
+			log.Println(err)
-			return
 		}
 
-		fmt.Println(generatedID)
+		sw.Stop()
 		return
 	}))
 	http.HandleFunc("/links.json", func(w http.ResponseWriter, r *http.Request) {
@@ -1258,9 +1330,9 @@ func createSearchIndex(dataDir, indexName string) (bleve.Index, error) {
 		indexMapping := bleve.NewIndexMapping()
 		documentMapping := bleve.NewDocumentMapping()
 
-		nameFieldMapping := bleve.NewTextFieldMapping()
+		pageFieldMapping := bleve.NewTextFieldMapping()
-		nameFieldMapping.Store = true
+		pageFieldMapping.Store = true
-		documentMapping.AddFieldMappingsAt("name", nameFieldMapping)
+		documentMapping.AddFieldMappingsAt("page", pageFieldMapping)
 
 		titleFieldMapping := bleve.NewTextFieldMapping()
 		titleFieldMapping.Store = true
@@ -1270,6 +1342,15 @@ func createSearchIndex(dataDir, indexName string) (bleve.Index, error) {
 		linkFieldMapping.Store = true
 		documentMapping.AddFieldMappingsAt("link", linkFieldMapping)
 
+		textFieldMapping := bleve.NewTextFieldMapping()
+		textFieldMapping.Store = true
+		documentMapping.AddFieldMappingsAt("text", textFieldMapping)
+
+		dateFieldMapping := bleve.NewDateTimeFieldMapping()
+		dateFieldMapping.Store = false
+		dateFieldMapping.Index = true
+		documentMapping.AddFieldMappingsAt("date", dateFieldMapping)
+
 		indexMapping.AddDocumentMapping("block", documentMapping)
 
 		searchIndex, err := bleve.New(indexDir, indexMapping)

search.go

@@ -25,9 +25,10 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"time"
 
-	"github.com/blevesearch/bleve"
+	"github.com/blevesearch/bleve/v2"
-	"github.com/blevesearch/bleve/mapping"
+	"github.com/blevesearch/bleve/v2/mapping"
 	"github.com/iancoleman/strcase"
 )
 
@@ -53,6 +54,7 @@ type searchObject struct {
 	Refs   []nameLine             `json:"refs"`
 	Meta   map[string]interface{} `json:"meta"`
 	Links  []ParsedLink           `json:"links"`
+	Dates  []time.Time            `json:"dates"`
 }
 
 func NewSearchHandler(searchIndex bleve.Index) (http.Handler, error) {
@@ -104,6 +106,9 @@ func (s *searchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 		if r.PostForm.Get("reset") == "1" {
+			var sw stopwatch
+			sw.Start("full reset")
+			defer sw.Stop()
 
 			refs := make(Refs)
 			mp := NewFilePages("data", nil)
@@ -122,6 +127,8 @@ func (s *searchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 				}
 			}
 
+			sw.Lap("save blocks from pages")
+
 			// Reload all pages
 			pages, err = mp.AllPages()
 			if err != nil {
@@ -138,21 +145,23 @@ func (s *searchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 				}
 			}
 
-			log.Println("saveLinks")
+			sw.Lap("process backrefs for pages")
+
 			err = saveLinks(mp)
 			if err != nil {
 				log.Printf("error while saving links %v", err)
 				http.Error(w, err.Error(), 500)
 				return
 			}
+			sw.Lap("save links")
 
-			log.Println("saveBackrefs")
 			err = saveBackrefs("data/backrefs.json", refs)
 			if err != nil {
 				log.Printf("error while saving backrefs %v", err)
 				http.Error(w, err.Error(), 500)
 				return
 			}
+			sw.Lap("save backrefs")
 
 			err = os.RemoveAll("data/_tmp_index")
 			if err != nil {
@@ -161,28 +170,12 @@ func (s *searchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 
-			index, err := createSearchIndex("data", "_tmp_index")
+			_, err = createSearchIndex("data", "_tmp_index")
 			if err != nil {
 				http.Error(w, err.Error(), 500)
 				return
 			}
 
-			for _, page := range pages {
-				searchObjects, err := createSearchObjects(page.Name)
-				if err != nil {
-					log.Printf("error while creating search object %s: %v", page.Title, err)
-					continue
-				}
-
-				for _, so := range searchObjects {
-					err = index.Index(so.ID, so)
-					if err != nil {
-						log.Printf("error while indexing %s: %v", page.Title, err)
-						continue
-					}
-				}
-			}
-
 			err = os.Rename("data/_page-index", "data/_page-index-old")
 			if err != nil {
 				log.Printf("error while resetting index: %v", err)
@@ -201,6 +194,7 @@ func (s *searchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 				http.Error(w, err.Error(), 500)
 				return
 			}
+			sw.Lap("indexing")
 
 			enc := json.NewEncoder(w)
 			enc.SetIndent("", "    ")
@@ -219,7 +213,7 @@ func (s *searchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	sr := bleve.NewSearchRequest(q)
 	sr.IncludeLocations = false
 	sr.Size = 25
-	sr.Fields = []string{"page", "title", "text"}
+	sr.Fields = []string{"page", "title", "text", "date", "parent"}
 	sr.Highlight = bleve.NewHighlightWithStyle("html")
 	sr.Highlight.AddField("text")
 	results, err := s.searchIndex.Search(sr)
@@ -236,10 +230,15 @@ func (s *searchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 type pageBlock struct {
 	ID     string      `json:"id"`
+	Parent string      `json:"parent"`
 	Title  string      `json:"title"`
 	Page   string      `json:"page"`
 	Text   string      `json:"text"`
-	Link  string `json:"link"`
+	Link   []string    `json:"link"`
+	Tag    []string    `json:"tag"`
+	Date   []time.Time `json:"date"`
+	Key    string      `json:"key"`
+	Value  string      `json:"value"`
 }
 
 func (p pageBlock) Type() string {
@@ -247,11 +246,19 @@ func (p pageBlock) Type() string {
 }
 
 func createSearchObjects(rootBlockID string) ([]pageBlock, error) {
+	log.Println("createSearchObjects", rootBlockID)
 	blocks, err := loadBlocks("data", rootBlockID)
 	if err != nil {
 		return nil, err
 	}
 
+	if len(blocks.Parents) > 0 {
+		page := blocks.Parents[len(blocks.Parents)-1]
+		if page != rootBlockID {
+			blocks, err = loadBlocks("data", page)
+		}
+	}
+
 	var pageBlocks []pageBlock
 
 	queue := []string{blocks.PageID}
@@ -262,28 +269,48 @@ func createSearchObjects(rootBlockID string) ([]pageBlock, error) {
 
 		links, err := ParseLinks(current, blocks.Texts[current])
 		if err != nil {
-			continue
+			log.Println("ParseLinks", err)
+			links = nil
 		}
 
-		if len(links) == 0 {
+		var linkNames []string
-			pageBlocks = append(pageBlocks, pageBlock{
-				ID:    current,
-				Title: blocks.Texts[blocks.PageID],
-				Page:  blocks.PageID,
-				Text:  blocks.Texts[current],
-				Link:  "",
-			})
-		} else {
 		for _, link := range links {
-				pageBlocks = append(pageBlocks, pageBlock{
+			linkNames = append(linkNames, link.Name)
+		}
+
+		tags, err := ParseTags(blocks.Texts[current])
+		if err != nil {
+			log.Println("ParseTags", err)
+			tags = nil
+		}
+
+		dates, err := ParseDates(blocks.Texts[current])
+		if err != nil {
+			log.Println("ParseDates", err)
+			dates = nil
+		}
+
+		pageDate, err := ParseDatePageName(blocks.Texts[blocks.PageID])
+		if err == nil {
+			dates = append(dates, pageDate)
+		}
+
+		block := pageBlock{
 			ID:     current,
+			Parent: blocks.ParentID,
 			Title:  blocks.Texts[blocks.PageID],
 			Page:   blocks.PageID,
 			Text:   blocks.Texts[current],
-					Link:  link.Name,
+			Link:   linkNames,
-				})
+			Tag:    tags,
+			Date:   dates,
 		}
+
+		if kvpair := strings.SplitN(blocks.Texts[current], "::", 2); len(kvpair) == 2 {
+			block.Key = strings.TrimSpace(kvpair[0])
+			block.Value = strings.TrimSpace(kvpair[1])
 		}
+		pageBlocks = append(pageBlocks, block)
 
 		queue = append(queue, blocks.Children[current]...)
 	}
@@ -400,9 +427,19 @@ func createStructuredFormat(page Page) (searchObject, error) {
 			}
 
 			so.Links = append(so.Links, links...)
+
+			dates, err := ParseDates(li.Text)
+			if err != nil {
+				dates = nil
+			}
+			so.Dates = append(so.Dates, dates...)
 		}
 	}
 
+	date, err := ParseDatePageName(so.Title)
+	if err == nil {
+		so.Dates = append(so.Dates, date)
+	}
 	// merge up
 	for len(parents) > 1 {
 		par := parents[len(parents)-1]

templates/edit.html

@@ -18,23 +18,6 @@
                 <input type="hidden" name="p" value="{{ .Name }}"/>
                 {{ .Editor }}
             </form>
-
-            {{ if .Backrefs }}
-                <div class="backrefs content">
-                    <h3>Linked references</h3>
-                    <ul>
-                        {{ range $name, $refs := .Backrefs }}
-                            <li><a href="/edit/{{ $name }}">{{ (index $refs 0).Title }}</a>
-                                <ul>
-                                    {{ range $ref := $refs }}
-                                        <li>{{ $ref.LineEditHTML }}</li>
-                                    {{ end }}
-                                </ul>
-                            </li>
-                        {{ end }}
-                    </ul>
-                </div>
-            {{ end }}
         </div>
     </div>
 {{ end }}

templates/layout_no_sidebar.html

@@ -0,0 +1,83 @@
+<!doctype html>
+<html>
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport"
+          content="width=device-width, user-scalable=no, initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0">
+    <meta http-equiv="X-UA-Compatible" content="ie=edge">
+    <link rel="redirect_uri" href="{{ .RedirectURI }}" />
+    <link rel="stylesheet" href="/public/main.css" />
+    <link rel="stylesheet" href="/public/styles.css" />
+<!--     <link rel="stylesheet" href="https://unpkg.com/prismjs@1.20.0/themes/prism-tomorrow.css"> -->
+<!--     <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css"/> -->
+    <title>{{ .Title }} - Wiki</title>
+    {{ block "content_head" . }} {{ end }}
+    <style>
+    </style>
+    <style>
+    </style>
+</head>
+<body data-base-url="{{ .BaseURL }}">
+<div>
+    <nav class="navbar" role="navigation" aria-label="main navigation">
+        <div class="navbar-brand">
+            <a class="navbar-item" href="/">
+                Wiki
+            </a>
+
+            <a role="button" class="navbar-burger burger" aria-label="menu" aria-expanded="false"
+               data-target="navbarBasicExample">
+                <span aria-hidden="true"></span>
+                <span aria-hidden="true"></span>
+                <span aria-hidden="true"></span>
+            </a>
+        </div>
+
+        <div id="navbarBasicExample" class="navbar-menu">
+            <div class="navbar-start">
+                {{ block "navbar" . }}{{ end }}
+            </div>
+        </div>
+    </nav>
+
+    <div class="searchbar">
+        <div class="field">
+            <p class="control">
+                <input class="search input" id="search-input" type="text" placeholder="Find a page">
+            </p>
+            <div id="autocomplete" class="hide keyboard-list" tabindex="0"></div>
+        </div>
+    </div>
+
+    <div style="padding:0 32px;">
+        <section class="section">
+            {{ template "content" . }}
+        </section>
+    </div>
+
+
+    <div class="footer">
+    <div class="h-app">
+        <a href="/" class="u-url p-name">Wiki</a>
+        &mdash; created by <a href="https://peterstuifzand.nl/">Peter Stuifzand</a>
+    </div>
+    </div>
+</div>
+<div id="link-complete" class="hide keyboard-list"></div>
+{{ block "footer_scripts" . }}
+{{ end }}
+<div id="result-template" class="hide">
+    <ul>
+        [[#results]]
+        <li><a href="/edit/[[ref]]">[[title]] <div>[[&amp; text]]</div></a></li>
+        [[/results]]
+        [[^results]]
+        <li>No results</li>
+        [[/results]]
+        <li><a href="/edit/[[page]]">Create a page</a></li>
+    </ul>
+</div>
+<script async src="/public/vendors.js"></script>
+<script async src="/public/main.js"></script>
+</body>
+</html>

templates/sidebar-right.html

@@ -1,12 +1,48 @@
 {{ define "sidebar-right" }}
 <div class="sidebar-right">
+    <div class="tab-bar">
+        <div class="tab tab-active" data-target="#tab-calendar">
+            <span>Calendar</span>
+        </div>
+        <div class="tab" data-target="#tab-graph">
+            <span>Graph</span>
+        </div>
+        <div class="tab" data-target="#tab-linked-refs">
+            <span>Backlinks</span>
+        </div>
+    </div>
+    <div class="tabs">
+        <div class="tab-page" id="tab-calendar">
             {{block "calendar" .}}
             {{end}}
+        </div>
 
+        <div class="tab-page" id="tab-graph">
             <div class="graph-view">
                 {{ if .ShowGraph }}
                     <div class="graph-network" data-name="{{ .Name }}" style="height:80vh; top:0; position: sticky"></div>
                 {{ end }}
             </div>
         </div>
+
+        <div class="tab-page" id="tab-linked-refs">
+            {{ if .Backrefs }}
+                <div class="backrefs content">
+                    <h3>Linked references</h3>
+                    <ul>
+                        {{ range $name, $refs := .Backrefs }}
+                            <li><a href="/edit/{{ $name }}">{{ (index $refs 0).Title }}</a>
+                                <ul>
+                                    {{ range $ref := $refs }}
+                                        <li>{{ $ref.LineEditHTML }}</li>
+                                    {{ end }}
+                                </ul>
+                            </li>
+                        {{ end }}
+                    </ul>
+                </div>
+            {{ end }}
+        </div>
+    </div>
+</div>
 {{ end }}

templates/view.html

@@ -22,11 +22,30 @@
                 </ul>
             </div>
         </div>
+    </div>
 
-        <div class="column">
+    <div class="modal review-modal">
-            {{ if .ShowGraph }}
+        <div class="modal-background"></div>
-            <div class="graph-network" data-name="{{ .Name }}" style="height:80vh; top:0; position: sticky"></div>
+        <div class="modal-card">
-            {{ end }}
+            <header class="modal-card-head">
+              <p class="modal-card-title">Review</p>
+              <button class="delete" aria-label="close"></button>
+            </header>
+
+            <section class="modal-card-body" style="min-height: 400px">
+              <h3 class="block-title is-title"></h3>
+                <textarea class="textarea block-text" style="height:380px"></textarea>
+
+                <div class="end-of-review hide">All tasks are reviewed.</p>
+            </section>
+
+            <div class="modal-card-foot" style="justify-content: center">
+                <button class="button normal is-danger review" data-review="again">Again</button>
+                <button class="button hard is-warning review" data-review="soon">Soon</button>
+                <button class="button easy is-success review" data-review="later">Later</button>
+                <button class="button easy is-default review" data-review="never">Never</button>
+                <button class="button end-of-review hide close">Close</button>
+            </div>
         </div>
     </div>
 {{ end }}
@@ -38,6 +57,8 @@
         <a href="/recent/" class="navbar-item">Recent Changes</a>
         <a href="/graph/" class="navbar-item">Graph</a>
         <a href="/{{ .TodayPage }}" class="navbar-item">Today</a>
+        <a href="" class="navbar-item start-review">Review</a>
+        <a href="" class="navbar-item start-sr">SR</a>
         <a href="/auth/logout" class="navbar-item">Logout</a>
         <span class="navbar-item"><b>{{ $.Session.Me }}</b></span>
     {{ else }}
@@ -50,9 +71,5 @@
         .edit {
             color: red;
         }
-
-        .tag {
-            color: #444;
-        }
     </style>
 {{ end }}

util.go

@@ -29,6 +29,8 @@ import (
 	"strconv"
 	"strings"
 	"time"
+
+	"p83.nl/go/wiki/link"
 )
 
 var (
@@ -70,6 +72,25 @@ func RandStringBytes(n int) string {
 	return string(b)
 }
 
+type DateLink struct {
+	Link string
+	Date time.Time
+}
+
+func ParseDates(content string) ([]time.Time, error) {
+	links := link.FindAllLinks(content)
+	var result []time.Time
+	for _, linkName := range links {
+		date, err := ParseDatePageName(linkName)
+		if err != nil {
+			continue
+		}
+		result = append(result, date)
+	}
+
+	return result, nil
+}
+
 func ParseLinks(blockId string, content string) ([]ParsedLink, error) {
 	hrefRE := regexp.MustCompile(`(#?\[\[\s*([^\]]+)\s*\]\])`)
 	// keywordsRE := regexp.MustCompile(`(\w+)::`)
@@ -110,6 +131,33 @@ func ParseLinks(blockId string, content string) ([]ParsedLink, error) {
 	return result, nil
 }
 
+func ParseTags(content string) ([]string, error) {
+	linkRE := regexp.MustCompile(`(#\[\[\s*([^\]]+)\s*\]\])`)
+	tagRE := regexp.MustCompile(`#([^ ]+)`)
+
+	scanner := bufio.NewScanner(strings.NewReader(content))
+	scanner.Split(bufio.ScanLines)
+
+	var result []string
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		links := linkRE.FindAllStringSubmatch(line, -1)
+		for _, matches := range links {
+			linkText := matches[0]
+			linkText = strings.TrimPrefix(linkText, "#[[")
+			linkText = strings.TrimSuffix(linkText, "]]")
+			linkText = strings.TrimSpace(linkText)
+			result = append(result, linkText)
+		}
+		for _, matches := range tagRE.FindAllStringSubmatch(line, -1) {
+			result = append(result, matches[1])
+		}
+	}
+
+	return result, nil
+}
+
 func cleanNameURL(name string) string {
 	return strings.Replace(name, " ", "_", -1)
 }
@@ -120,14 +168,23 @@ func cleanTitle(name string) string {
 
 type stopwatch struct {
 	start   time.Time
+	lastLap time.Time
 	label   string
 }
 
 func (sw *stopwatch) Start(label string) {
 	sw.start = time.Now()
+	sw.lastLap = time.Now()
 	sw.label = label
 }
 
+func (sw *stopwatch) Lap(label string) {
+	now := time.Now()
+	d := now.Sub(sw.lastLap)
+	log.Printf("%-20s: %s\n", label, d.String())
+	sw.lastLap = now
+}
+
 func (sw *stopwatch) Stop() {
 	endTime := time.Now()
 	d := endTime.Sub(sw.start)
@@ -176,7 +233,7 @@ func parseMonth(month string) (time.Month, error) {
 }
 
 func ParseDatePageName(name string) (time.Time, error) {
-	if matches := niceDateParseRE.FindStringSubmatch(name); matches != nil {
+	if matches := niceDateParseRE.FindStringSubmatch(strings.Replace(name, " ", "_", -1)); matches != nil {
 		day, err := strconv.Atoi(matches[1])
 		if err != nil {
 			return time.Time{}, fmt.Errorf("%q: %s: %w", name, err, ParseFailed)
@@ -189,7 +246,7 @@ func ParseDatePageName(name string) (time.Time, error) {
 		if err != nil {
 			return time.Time{}, fmt.Errorf("%q: %s: %w", name, err, ParseFailed)
 		}
-		return time.Date(year, month, day, 0, 0, 0, 0, time.Local), nil
+		return time.Date(year, month, day, 0, 0, 0, 0, time.UTC), nil
 	}
 	return time.Time{}, fmt.Errorf("%q: invalid syntax: %w", name, ParseFailed)
 }