Merge branch 'master' into master

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -24,7 +24,7 @@ typically be found at `/etc/gitea/conf/app.ini`.
 
 The defaults provided here are best-effort (not built automatically). They are
 accurately recorded in [app.ini.sample](https://github.com/go-gitea/gitea/blob/master/custom/conf/app.ini.sample)
-(s/master/<tag|release\>). Any string in the format `%(X)s` is a feature powered
+(s/master/\<tag|release\>). Any string in the format `%(X)s` is a feature powered
 by [ini](https://github.com/go-ini/ini/#recursive-values), for reading values recursively.
 
 Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
@@ -118,7 +118,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `HOST`: **127.0.0.1:3306**: Database host address and port.
 - `NAME`: **gitea**: Database name.
 - `USER`: **root**: Database username.
-- `PASSWD`: **\<empty\>**: Database user password. Use \`your password\` for quoting if you use special characters in the password. 
+- `PASSWD`: **\<empty\>**: Database user password. Use \`your password\` for quoting if you use special characters in the password.
 - `SSL_MODE`: **disable**: For PostgreSQL only.
 - `PATH`: **data/gitea.db**: For SQLite3 only, the database file path.
 
@@ -128,7 +128,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space).
 - `REPO_INDEXER_PATH`: **indexers/repos.bleve**: Index file used for code search.
 - `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request.
-- `MAX_FILE_SIZE`: **1048576**: Maximum size in bytes of each index files.
+- `MAX_FILE_SIZE`: **1048576**: Maximum size in bytes of files to be indexed.
 
 ## Security (`security`)
 

integrations/git_test.go

@@ -13,6 +13,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"testing"
 	"time"
@@ -134,7 +135,7 @@ func TestGit(t *testing.T) {
 
 			//Setup key
 			keyFile := filepath.Join(setting.AppDataPath, "my-testing-key")
-			_, _, err := com.ExecCmd("ssh-keygen", "-f", keyFile, "-t", "rsa", "-N", "")
+			err := exec.Command("ssh-keygen", "-f", keyFile, "-t", "rsa", "-N", "").Run()
 			assert.NoError(t, err)
 			defer os.RemoveAll(keyFile)
 			defer os.RemoveAll(keyFile + ".pub")
@@ -152,13 +153,10 @@ func TestGit(t *testing.T) {
 			session.MakeRequest(t, req, http.StatusCreated)
 
 			//Setup ssh wrapper
-			sshWrapper, err := ioutil.TempFile(setting.AppDataPath, "tmp-ssh-wrapper")
-			sshWrapper.WriteString("#!/bin/sh\n\n")
-			sshWrapper.WriteString("ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i \"" + filepath.Join(setting.AppWorkPath, keyFile) + "\" $* \n\n")
-			err = sshWrapper.Chmod(os.ModePerm)
-			assert.NoError(t, err)
-			sshWrapper.Close()
-			defer os.RemoveAll(sshWrapper.Name())
+			os.Setenv("GIT_SSH_COMMAND",
+				"ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "+
+					filepath.Join(setting.AppWorkPath, keyFile))
+			os.Setenv("GIT_SSH_VARIANT", "ssh")
 
 			//Setup clone folder
 			dstPath, err := ioutil.TempDir("", "repo-tmp-18")
@@ -181,7 +179,7 @@ func TestGit(t *testing.T) {
 				})
 				//TODO get url from api
 				t.Run("Clone", func(t *testing.T) {
-					_, err = git.NewCommand("clone").AddArguments("--config", "core.sshCommand="+filepath.Join(setting.AppWorkPath, sshWrapper.Name()), u.String(), dstPath).Run()
+					_, err = git.NewCommand("clone").AddArguments(u.String(), dstPath).Run()
 					assert.NoError(t, err)
 					assert.True(t, com.IsExist(filepath.Join(dstPath, "README.md")))
 				})
@@ -196,8 +194,6 @@ func TestGit(t *testing.T) {
 				})
 			})
 			t.Run("LFS", func(t *testing.T) {
-				os.Setenv("GIT_SSH_COMMAND", filepath.Join(setting.AppWorkPath, sshWrapper.Name())) //TODO remove when fixed https://github.com/git-lfs/git-lfs/issues/2215
-				defer os.Unsetenv("GIT_SSH_COMMAND")
 				t.Run("PushCommit", func(t *testing.T) {
 					//Setup git LFS
 					_, err = git.NewCommand("lfs").AddArguments("install").RunInDir(dstPath)

main.go

@@ -21,7 +21,7 @@ import (
 )
 
 // Version holds the current Gitea version
-var Version = "1.4.0-dev"
+var Version = "1.5.0-dev"
 
 // Tags holds the build tags used
 var Tags = ""

models/error.go

@@ -216,6 +216,21 @@ func (err ErrWikiReservedName) Error() string {
 	return fmt.Sprintf("wiki title is reserved: %s", err.Title)
 }
 
+// ErrWikiInvalidFileName represents an invalid wiki file name.
+type ErrWikiInvalidFileName struct {
+	FileName string
+}
+
+// IsErrWikiInvalidFileName checks if an error is an ErrWikiInvalidFileName.
+func IsErrWikiInvalidFileName(err error) bool {
+	_, ok := err.(ErrWikiInvalidFileName)
+	return ok
+}
+
+func (err ErrWikiInvalidFileName) Error() string {
+	return fmt.Sprintf("Invalid wiki filename: %s", err.FileName)
+}
+
 // __________     ___.   .__  .__          ____  __.
 // \______   \__ _\_ |__ |  | |__| ____   |    |/ _|____ ___.__.
 //  |     ___/  |  \ __ \|  | |  |/ ___\  |      <_/ __ <   |  |

models/issue_indexer.go

@@ -53,7 +53,7 @@ func populateIssueIndexer() error {
 				return err
 			}
 			for _, issue := range issues {
-				if err := batch.Add(issue.update()); err != nil {
+				if err := issue.update().AddToFlushingBatch(batch); err != nil {
 					return err
 				}
 			}
@@ -78,7 +78,7 @@ func processIssueIndexerUpdateQueue() {
 		issue, err := GetIssueByID(issueID)
 		if err != nil {
 			log.Error(4, "GetIssueByID: %v", err)
-		} else if err = batch.Add(issue.update()); err != nil {
+		} else if err = issue.update().AddToFlushingBatch(batch); err != nil {
 			log.Error(4, "IssueIndexer: %v", err)
 		}
 	}

models/repo_indexer.go

@@ -5,9 +5,7 @@
 package models
 
 import (
-	"io/ioutil"
-	"os"
-	"path"
+	"fmt"
 	"strconv"
 	"strings"
 
@@ -17,7 +15,7 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 
-	"github.com/Unknwon/com"
+	"github.com/ethantkoenig/rupture"
 )
 
 // RepoIndexerStatus status of a repo's entry in the repo indexer
@@ -132,7 +130,11 @@ func populateRepoIndexer(maxRepoID int64) {
 }
 
 func updateRepoIndexer(repo *Repository) error {
-	changes, err := getRepoChanges(repo)
+	sha, err := getDefaultBranchSha(repo)
+	if err != nil {
+		return err
+	}
+	changes, err := getRepoChanges(repo, sha)
 	if err != nil {
 		return err
 	} else if changes == nil {
@@ -140,12 +142,12 @@ func updateRepoIndexer(repo *Repository) error {
 	}
 
 	batch := indexer.RepoIndexerBatch()
-	for _, filename := range changes.UpdatedFiles {
-		if err := addUpdate(filename, repo, batch); err != nil {
+	for _, update := range changes.Updates {
+		if err := addUpdate(update, repo, batch); err != nil {
 			return err
 		}
 	}
-	for _, filename := range changes.RemovedFiles {
+	for _, filename := range changes.RemovedFilenames {
 		if err := addDelete(filename, repo, batch); err != nil {
 			return err
 		}
@@ -153,110 +155,151 @@ func updateRepoIndexer(repo *Repository) error {
 	if err = batch.Flush(); err != nil {
 		return err
 	}
-	return updateLastIndexSync(repo)
+	return repo.updateIndexerStatus(sha)
 }
 
 // repoChanges changes (file additions/updates/removals) to a repo
 type repoChanges struct {
-	UpdatedFiles []string
-	RemovedFiles []string
+	Updates          []fileUpdate
+	RemovedFilenames []string
+}
+
+type fileUpdate struct {
+	Filename string
+	BlobSha  string
+}
+
+func getDefaultBranchSha(repo *Repository) (string, error) {
+	stdout, err := git.NewCommand("show-ref", "-s", repo.DefaultBranch).RunInDir(repo.RepoPath())
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(stdout), nil
 }
 
 // getRepoChanges returns changes to repo since last indexer update
-func getRepoChanges(repo *Repository) (*repoChanges, error) {
-	repoWorkingPool.CheckIn(com.ToStr(repo.ID))
-	defer repoWorkingPool.CheckOut(com.ToStr(repo.ID))
-
-	if err := repo.UpdateLocalCopyBranch(""); err != nil {
-		return nil, err
-	} else if !git.IsBranchExist(repo.LocalCopyPath(), repo.DefaultBranch) {
-		// repo does not have any commits yet, so nothing to update
-		return nil, nil
-	} else if err = repo.UpdateLocalCopyBranch(repo.DefaultBranch); err != nil {
-		return nil, err
-	} else if err = repo.getIndexerStatus(); err != nil {
+func getRepoChanges(repo *Repository, revision string) (*repoChanges, error) {
+	if err := repo.getIndexerStatus(); err != nil {
 		return nil, err
 	}
 
 	if len(repo.IndexerStatus.CommitSha) == 0 {
-		return genesisChanges(repo)
+		return genesisChanges(repo, revision)
 	}
-	return nonGenesisChanges(repo)
+	return nonGenesisChanges(repo, revision)
 }
 
-func addUpdate(filename string, repo *Repository, batch *indexer.Batch) error {
-	filepath := path.Join(repo.LocalCopyPath(), filename)
-	if stat, err := os.Stat(filepath); err != nil {
+func addUpdate(update fileUpdate, repo *Repository, batch rupture.FlushingBatch) error {
+	stdout, err := git.NewCommand("cat-file", "-s", update.BlobSha).
+		RunInDir(repo.RepoPath())
+	if err != nil {
 		return err
-	} else if stat.Size() > setting.Indexer.MaxIndexerFileSize {
-		return nil
-	} else if stat.IsDir() {
-		// file could actually be a directory, if it is the root of a submodule.
-		// We do not index submodule contents, so don't do anything.
+	}
+	if size, err := strconv.Atoi(strings.TrimSpace(stdout)); err != nil {
+		return fmt.Errorf("Misformatted git cat-file output: %v", err)
+	} else if int64(size) > setting.Indexer.MaxIndexerFileSize {
 		return nil
 	}
-	fileContents, err := ioutil.ReadFile(filepath)
+
+	fileContents, err := git.NewCommand("cat-file", "blob", update.BlobSha).
+		RunInDirBytes(repo.RepoPath())
 	if err != nil {
 		return err
 	} else if !base.IsTextFile(fileContents) {
 		return nil
 	}
-	return batch.Add(indexer.RepoIndexerUpdate{
-		Filepath: filename,
+	indexerUpdate := indexer.RepoIndexerUpdate{
+		Filepath: update.Filename,
 		Op:       indexer.RepoIndexerOpUpdate,
 		Data: &indexer.RepoIndexerData{
 			RepoID:  repo.ID,
 			Content: string(fileContents),
 		},
-	})
+	}
+	return indexerUpdate.AddToFlushingBatch(batch)
 }
 
-func addDelete(filename string, repo *Repository, batch *indexer.Batch) error {
-	return batch.Add(indexer.RepoIndexerUpdate{
+func addDelete(filename string, repo *Repository, batch rupture.FlushingBatch) error {
+	indexerUpdate := indexer.RepoIndexerUpdate{
 		Filepath: filename,
 		Op:       indexer.RepoIndexerOpDelete,
 		Data: &indexer.RepoIndexerData{
 			RepoID: repo.ID,
 		},
-	})
+	}
+	return indexerUpdate.AddToFlushingBatch(batch)
 }
 
-// genesisChanges get changes to add repo to the indexer for the first time
-func genesisChanges(repo *Repository) (*repoChanges, error) {
-	var changes repoChanges
-	stdout, err := git.NewCommand("ls-files").RunInDir(repo.LocalCopyPath())
-	if err != nil {
-		return nil, err
-	}
-	for _, line := range strings.Split(stdout, "\n") {
-		filename := strings.TrimSpace(line)
-		if len(filename) == 0 {
+// parseGitLsTreeOutput parses the output of a `git ls-tree -r --full-name` command
+func parseGitLsTreeOutput(stdout string) ([]fileUpdate, error) {
+	lines := strings.Split(stdout, "\n")
+	updates := make([]fileUpdate, 0, len(lines))
+	for _, line := range lines {
+		// expect line to be "<mode> <object-type> <object-sha>\t<filename>"
+		line = strings.TrimSpace(line)
+		if len(line) == 0 {
 			continue
-		} else if filename[0] == '"' {
+		}
+		firstSpaceIndex := strings.IndexByte(line, ' ')
+		if firstSpaceIndex < 0 {
+			log.Error(4, "Misformatted git ls-tree output: %s", line)
+			continue
+		}
+		tabIndex := strings.IndexByte(line, '\t')
+		if tabIndex < 42+firstSpaceIndex || tabIndex == len(line)-1 {
+			log.Error(4, "Misformatted git ls-tree output: %s", line)
+			continue
+		}
+		if objectType := line[firstSpaceIndex+1 : tabIndex-41]; objectType != "blob" {
+			// submodules appear as commit objects, we do not index submodules
+			continue
+		}
+
+		blobSha := line[tabIndex-40 : tabIndex]
+		filename := line[tabIndex+1:]
+		if filename[0] == '"' {
+			var err error
 			filename, err = strconv.Unquote(filename)
 			if err != nil {
 				return nil, err
 			}
 		}
-		changes.UpdatedFiles = append(changes.UpdatedFiles, filename)
+		updates = append(updates, fileUpdate{
+			Filename: filename,
+			BlobSha:  blobSha,
+		})
 	}
-	return &changes, nil
+	return updates, nil
+}
+
+// genesisChanges get changes to add repo to the indexer for the first time
+func genesisChanges(repo *Repository, revision string) (*repoChanges, error) {
+	var changes repoChanges
+	stdout, err := git.NewCommand("ls-tree", "--full-tree", "-r", revision).
+		RunInDir(repo.RepoPath())
+	if err != nil {
+		return nil, err
+	}
+	changes.Updates, err = parseGitLsTreeOutput(stdout)
+	return &changes, err
 }
 
 // nonGenesisChanges get changes since the previous indexer update
-func nonGenesisChanges(repo *Repository) (*repoChanges, error) {
+func nonGenesisChanges(repo *Repository, revision string) (*repoChanges, error) {
 	diffCmd := git.NewCommand("diff", "--name-status",
-		repo.IndexerStatus.CommitSha, "HEAD")
-	stdout, err := diffCmd.RunInDir(repo.LocalCopyPath())
+		repo.IndexerStatus.CommitSha, revision)
+	stdout, err := diffCmd.RunInDir(repo.RepoPath())
 	if err != nil {
 		// previous commit sha may have been removed by a force push, so
 		// try rebuilding from scratch
+		log.Warn("git diff: %v", err)
 		if err = indexer.DeleteRepoFromIndexer(repo.ID); err != nil {
 			return nil, err
 		}
-		return genesisChanges(repo)
+		return genesisChanges(repo, revision)
 	}
 	var changes repoChanges
+	updatedFilenames := make([]string, 0, 10)
 	for _, line := range strings.Split(stdout, "\n") {
 		line = strings.TrimSpace(line)
 		if len(line) == 0 {
@@ -274,23 +317,22 @@ func nonGenesisChanges(repo *Repository) (*repoChanges, error) {
 
 		switch status := line[0]; status {
 		case 'M', 'A':
-			changes.UpdatedFiles = append(changes.UpdatedFiles, filename)
+			updatedFilenames = append(updatedFilenames, filename)
 		case 'D':
-			changes.RemovedFiles = append(changes.RemovedFiles, filename)
+			changes.RemovedFilenames = append(changes.RemovedFilenames, filename)
 		default:
 			log.Warn("Unrecognized status: %c (line=%s)", status, line)
 		}
 	}
-	return &changes, nil
-}
 
-func updateLastIndexSync(repo *Repository) error {
-	stdout, err := git.NewCommand("rev-parse", "HEAD").RunInDir(repo.LocalCopyPath())
+	cmd := git.NewCommand("ls-tree", "--full-tree", revision, "--")
+	cmd.AddArguments(updatedFilenames...)
+	stdout, err = cmd.RunInDir(repo.RepoPath())
 	if err != nil {
-		return err
+		return nil, err
 	}
-	sha := strings.TrimSpace(stdout)
-	return repo.updateIndexerStatus(sha)
+	changes.Updates, err = parseGitLsTreeOutput(stdout)
+	return &changes, err
 }
 
 func processRepoIndexerOperationQueue() {

models/user.go

@@ -145,7 +145,7 @@ func (u *User) BeforeUpdate() {
 		if len(u.AvatarEmail) == 0 {
 			u.AvatarEmail = u.Email
 		}
-		if len(u.AvatarEmail) > 0 {
+		if len(u.AvatarEmail) > 0 && u.Avatar == "" {
 			u.Avatar = base.HashEmail(u.AvatarEmail)
 		}
 	}
@@ -299,7 +299,9 @@ func (u *User) generateRandomAvatar(e Engine) error {
 	}
 	// NOTICE for random avatar, it still uses id as avatar name, but custom avatar use md5
 	// since random image is not a user's photo, there is no security for enumable
-	u.Avatar = fmt.Sprintf("%d", u.ID)
+	if u.Avatar == "" {
+		u.Avatar = fmt.Sprintf("%d", u.ID)
+	}
 	if err = os.MkdirAll(filepath.Dir(u.CustomAvatarPath()), os.ModePerm); err != nil {
 		return fmt.Errorf("MkdirAll: %v", err)
 	}

models/wiki.go

@@ -45,7 +45,7 @@ func WikiNameToFilename(name string) string {
 // WikiFilenameToName converts a wiki filename to its corresponding page name.
 func WikiFilenameToName(filename string) (string, error) {
 	if !strings.HasSuffix(filename, ".md") {
-		return "", fmt.Errorf("Invalid wiki filename: %s", filename)
+		return "", ErrWikiInvalidFileName{filename}
 	}
 	basename := filename[:len(filename)-3]
 	unescaped, err := url.QueryUnescape(basename)

models/wiki_test.go

@@ -77,11 +77,14 @@ func TestWikiFilenameToName(t *testing.T) {
 	for _, badFilename := range []string{
 		"nofileextension",
 		"wrongfileextension.txt",
-		"badescaping%%.md",
 	} {
 		_, err := WikiFilenameToName(badFilename)
 		assert.Error(t, err)
+		assert.True(t, IsErrWikiInvalidFileName(err))
 	}
+	_, err := WikiFilenameToName("badescaping%%.md")
+	assert.Error(t, err)
+	assert.False(t, IsErrWikiInvalidFileName(err))
 }
 
 func TestWikiNameToFilenameToName(t *testing.T) {

modules/indexer/indexer.go

@@ -6,12 +6,17 @@ package indexer
 
 import (
 	"fmt"
+	"os"
 	"strconv"
 
+	"code.gitea.io/gitea/modules/setting"
+
 	"github.com/blevesearch/bleve"
 	"github.com/blevesearch/bleve/analysis/token/unicodenorm"
+	"github.com/blevesearch/bleve/index/upsidedown"
 	"github.com/blevesearch/bleve/mapping"
 	"github.com/blevesearch/bleve/search/query"
+	"github.com/ethantkoenig/rupture"
 )
 
 // indexerID a bleve-compatible unique identifier for an integer id
@@ -53,40 +58,36 @@ func addUnicodeNormalizeTokenFilter(m *mapping.IndexMappingImpl) error {
 	})
 }
 
-// Update represents an update to an indexer
-type Update interface {
-	addToBatch(batch *bleve.Batch) error
-}
-
 const maxBatchSize = 16
 
-// Batch batch of indexer updates that automatically flushes once it
-// reaches a certain size
-type Batch struct {
-	batch *bleve.Batch
-	index bleve.Index
-}
-
-// Add add update to batch, possibly flushing
-func (batch *Batch) Add(update Update) error {
-	if err := update.addToBatch(batch.batch); err != nil {
-		return err
+// openIndexer open the index at the specified path, checking for metadata
+// updates and bleve version updates.  If index needs to be created (or
+// re-created), returns (nil, nil)
+func openIndexer(path string, latestVersion int) (bleve.Index, error) {
+	_, err := os.Stat(setting.Indexer.IssuePath)
+	if err != nil && os.IsNotExist(err) {
+		return nil, nil
+	} else if err != nil {
+		return nil, err
 	}
-	return batch.flushIfFull()
-}
 
-func (batch *Batch) flushIfFull() error {
-	if batch.batch.Size() >= maxBatchSize {
-		return batch.Flush()
+	metadata, err := rupture.ReadIndexMetadata(path)
+	if err != nil {
+		return nil, err
+	}
+	if metadata.Version < latestVersion {
+		// the indexer is using a previous version, so we should delete it and
+		// re-populate
+		return nil, os.RemoveAll(path)
 	}
-	return nil
-}
 
-// Flush manually flush the batch, regardless of its size
-func (batch *Batch) Flush() error {
-	if err := batch.index.Batch(batch.batch); err != nil {
-		return err
+	index, err := bleve.Open(path)
+	if err != nil && err == upsidedown.IncompatibleVersion {
+		// the indexer was built with a previous version of bleve, so we should
+		// delete it and re-populate
+		return nil, os.RemoveAll(path)
+	} else if err != nil {
+		return nil, err
 	}
-	batch.batch.Reset()
-	return nil
+	return index, nil
 }

modules/indexer/issue.go

@@ -5,8 +5,6 @@
 package indexer
 
 import (
-	"os"
-
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 
@@ -14,12 +12,19 @@ import (
 	"github.com/blevesearch/bleve/analysis/analyzer/custom"
 	"github.com/blevesearch/bleve/analysis/token/lowercase"
 	"github.com/blevesearch/bleve/analysis/tokenizer/unicode"
-	"github.com/blevesearch/bleve/index/upsidedown"
+	"github.com/ethantkoenig/rupture"
 )
 
 // issueIndexer (thread-safe) index for searching issues
 var issueIndexer bleve.Index
 
+const (
+	issueIndexerAnalyzer = "issueIndexer"
+	issueIndexerDocType  = "issueIndexerDocType"
+
+	issueIndexerLatestVersion = 1
+)
+
 // IssueIndexerData data stored in the issue indexer
 type IssueIndexerData struct {
 	RepoID   int64
@@ -28,35 +33,33 @@ type IssueIndexerData struct {
 	Comments []string
 }
 
+// Type returns the document type, for bleve's mapping.Classifier interface.
+func (i *IssueIndexerData) Type() string {
+	return issueIndexerDocType
+}
+
 // IssueIndexerUpdate an update to the issue indexer
 type IssueIndexerUpdate struct {
 	IssueID int64
 	Data    *IssueIndexerData
 }
 
-func (update IssueIndexerUpdate) addToBatch(batch *bleve.Batch) error {
-	return batch.Index(indexerID(update.IssueID), update.Data)
+// AddToFlushingBatch adds the update to the given flushing batch.
+func (i IssueIndexerUpdate) AddToFlushingBatch(batch rupture.FlushingBatch) error {
+	return batch.Index(indexerID(i.IssueID), i.Data)
 }
 
-const issueIndexerAnalyzer = "issueIndexer"
-
 // InitIssueIndexer initialize issue indexer
 func InitIssueIndexer(populateIndexer func() error) {
-	_, err := os.Stat(setting.Indexer.IssuePath)
-	if err != nil && !os.IsNotExist(err) {
+	var err error
+	issueIndexer, err = openIndexer(setting.Indexer.IssuePath, issueIndexerLatestVersion)
+	if err != nil {
 		log.Fatal(4, "InitIssueIndexer: %v", err)
-	} else if err == nil {
-		issueIndexer, err = bleve.Open(setting.Indexer.IssuePath)
-		if err == nil {
-			return
-		} else if err != upsidedown.IncompatibleVersion {
-			log.Fatal(4, "InitIssueIndexer, open index: %v", err)
-		}
-		log.Warn("Incompatible bleve version, deleting and recreating issue indexer")
-		if err = os.RemoveAll(setting.Indexer.IssuePath); err != nil {
-			log.Fatal(4, "InitIssueIndexer: remove index, %v", err)
-		}
 	}
+	if issueIndexer != nil {
+		return
+	}
+
 	if err = createIssueIndexer(); err != nil {
 		log.Fatal(4, "InitIssuesIndexer: create index, %v", err)
 	}
@@ -70,9 +73,13 @@ func createIssueIndexer() error {
 	mapping := bleve.NewIndexMapping()
 	docMapping := bleve.NewDocumentMapping()
 
-	docMapping.AddFieldMappingsAt("RepoID", bleve.NewNumericFieldMapping())
+	numericFieldMapping := bleve.NewNumericFieldMapping()
+	numericFieldMapping.IncludeInAll = false
+	docMapping.AddFieldMappingsAt("RepoID", numericFieldMapping)
 
 	textFieldMapping := bleve.NewTextFieldMapping()
+	textFieldMapping.Store = false
+	textFieldMapping.IncludeInAll = false
 	docMapping.AddFieldMappingsAt("Title", textFieldMapping)
 	docMapping.AddFieldMappingsAt("Content", textFieldMapping)
 	docMapping.AddFieldMappingsAt("Comments", textFieldMapping)
@@ -89,7 +96,8 @@ func createIssueIndexer() error {
 	}
 
 	mapping.DefaultAnalyzer = issueIndexerAnalyzer
-	mapping.AddDocumentMapping("issues", docMapping)
+	mapping.AddDocumentMapping(issueIndexerDocType, docMapping)
+	mapping.AddDocumentMapping("_all", bleve.NewDocumentDisabledMapping())
 
 	var err error
 	issueIndexer, err = bleve.New(setting.Indexer.IssuePath, mapping)
@@ -97,11 +105,8 @@ func createIssueIndexer() error {
 }
 
 // IssueIndexerBatch batch to add updates to
-func IssueIndexerBatch() *Batch {
-	return &Batch{
-		batch: issueIndexer.NewBatch(),
-		index: issueIndexer,
-	}
+func IssueIndexerBatch() rupture.FlushingBatch {
+	return rupture.NewFlushingBatch(issueIndexer, maxBatchSize)
 }
 
 // SearchIssuesByKeyword searches for issues by given conditions.

modules/indexer/repo.go

@@ -5,7 +5,6 @@
 package indexer
 
 import (
-	"os"
 	"strings"
 
 	"code.gitea.io/gitea/modules/log"
@@ -15,10 +14,17 @@ import (
 	"github.com/blevesearch/bleve/analysis/analyzer/custom"
 	"github.com/blevesearch/bleve/analysis/token/camelcase"
 	"github.com/blevesearch/bleve/analysis/token/lowercase"
+	"github.com/blevesearch/bleve/analysis/token/unique"
 	"github.com/blevesearch/bleve/analysis/tokenizer/unicode"
+	"github.com/ethantkoenig/rupture"
 )
 
-const repoIndexerAnalyzer = "repoIndexerAnalyzer"
+const (
+	repoIndexerAnalyzer = "repoIndexerAnalyzer"
+	repoIndexerDocType  = "repoIndexerDocType"
+
+	repoIndexerLatestVersion = 1
+)
 
 // repoIndexer (thread-safe) index for repository contents
 var repoIndexer bleve.Index
@@ -40,6 +46,11 @@ type RepoIndexerData struct {
 	Content string
 }
 
+// Type returns the document type, for bleve's mapping.Classifier interface.
+func (d *RepoIndexerData) Type() string {
+	return repoIndexerDocType
+}
+
 // RepoIndexerUpdate an update to the repo indexer
 type RepoIndexerUpdate struct {
 	Filepath string
@@ -47,13 +58,14 @@ type RepoIndexerUpdate struct {
 	Data     *RepoIndexerData
 }
 
-func (update RepoIndexerUpdate) addToBatch(batch *bleve.Batch) error {
+// AddToFlushingBatch adds the update to the given flushing batch.
+func (update RepoIndexerUpdate) AddToFlushingBatch(batch rupture.FlushingBatch) error {
 	id := filenameIndexerID(update.Data.RepoID, update.Filepath)
 	switch update.Op {
 	case RepoIndexerOpUpdate:
 		return batch.Index(id, update.Data)
 	case RepoIndexerOpDelete:
-		batch.Delete(id)
+		return batch.Delete(id)
 	default:
 		log.Error(4, "Unrecognized repo indexer op: %d", update.Op)
 	}
@@ -62,48 +74,50 @@ func (update RepoIndexerUpdate) addToBatch(batch *bleve.Batch) error {
 
 // InitRepoIndexer initialize repo indexer
 func InitRepoIndexer(populateIndexer func() error) {
-	_, err := os.Stat(setting.Indexer.RepoPath)
+	var err error
+	repoIndexer, err = openIndexer(setting.Indexer.RepoPath, repoIndexerLatestVersion)
 	if err != nil {
-		if os.IsNotExist(err) {
-			if err = createRepoIndexer(); err != nil {
-				log.Fatal(4, "CreateRepoIndexer: %v", err)
-			}
-			if err = populateIndexer(); err != nil {
-				log.Fatal(4, "PopulateRepoIndex: %v", err)
-			}
-		} else {
-			log.Fatal(4, "InitRepoIndexer: %v", err)
-		}
-	} else {
-		repoIndexer, err = bleve.Open(setting.Indexer.RepoPath)
-		if err != nil {
-			log.Fatal(4, "InitRepoIndexer, open index: %v", err)
-		}
+		log.Fatal(4, "InitRepoIndexer: %v", err)
+	}
+	if repoIndexer != nil {
+		return
+	}
+
+	if err = createRepoIndexer(); err != nil {
+		log.Fatal(4, "CreateRepoIndexer: %v", err)
+	}
+	if err = populateIndexer(); err != nil {
+		log.Fatal(4, "PopulateRepoIndex: %v", err)
 	}
 }
 
 // createRepoIndexer create a repo indexer if one does not already exist
 func createRepoIndexer() error {
+	var err error
 	docMapping := bleve.NewDocumentMapping()
-	docMapping.AddFieldMappingsAt("RepoID", bleve.NewNumericFieldMapping())
+	numericFieldMapping := bleve.NewNumericFieldMapping()
+	numericFieldMapping.IncludeInAll = false
+	docMapping.AddFieldMappingsAt("RepoID", numericFieldMapping)
 
 	textFieldMapping := bleve.NewTextFieldMapping()
+	textFieldMapping.IncludeInAll = false
 	docMapping.AddFieldMappingsAt("Content", textFieldMapping)
 
 	mapping := bleve.NewIndexMapping()
-	if err := addUnicodeNormalizeTokenFilter(mapping); err != nil {
+	if err = addUnicodeNormalizeTokenFilter(mapping); err != nil {
 		return err
-	} else if err := mapping.AddCustomAnalyzer(repoIndexerAnalyzer, map[string]interface{}{
+	} else if err = mapping.AddCustomAnalyzer(repoIndexerAnalyzer, map[string]interface{}{
 		"type":          custom.Name,
 		"char_filters":  []string{},
 		"tokenizer":     unicode.Name,
-		"token_filters": []string{unicodeNormalizeName, camelcase.Name, lowercase.Name},
+		"token_filters": []string{unicodeNormalizeName, camelcase.Name, lowercase.Name, unique.Name},
 	}); err != nil {
 		return err
 	}
 	mapping.DefaultAnalyzer = repoIndexerAnalyzer
-	mapping.AddDocumentMapping("repo", docMapping)
-	var err error
+	mapping.AddDocumentMapping(repoIndexerDocType, docMapping)
+	mapping.AddDocumentMapping("_all", bleve.NewDocumentDisabledMapping())
+
 	repoIndexer, err = bleve.New(setting.Indexer.RepoPath, mapping)
 	return err
 }
@@ -121,11 +135,8 @@ func filenameOfIndexerID(indexerID string) string {
 }
 
 // RepoIndexerBatch batch to add updates to
-func RepoIndexerBatch() *Batch {
-	return &Batch{
-		batch: repoIndexer.NewBatch(),
-		index: repoIndexer,
-	}
+func RepoIndexerBatch() rupture.FlushingBatch {
+	return rupture.NewFlushingBatch(repoIndexer, maxBatchSize)
 }
 
 // DeleteRepoFromIndexer delete all of a repo's files from indexer
@@ -138,8 +149,7 @@ func DeleteRepoFromIndexer(repoID int64) error {
 	}
 	batch := RepoIndexerBatch()
 	for _, hit := range result.Hits {
-		batch.batch.Delete(hit.ID)
-		if err = batch.flushIfFull(); err != nil {
+		if err = batch.Delete(hit.ID); err != nil {
 			return err
 		}
 	}

modules/markup/html.go

@@ -38,9 +38,9 @@ var (
 	MentionPattern = regexp.MustCompile(`(\s|^|\W)@[0-9a-zA-Z-_\.]+`)
 
 	// IssueNumericPattern matches string that references to a numeric issue, e.g. #1287
-	IssueNumericPattern = regexp.MustCompile(`( |^|\()#[0-9]+\b`)
+	IssueNumericPattern = regexp.MustCompile(`( |^|\(|\[)#[0-9]+\b`)
 	// IssueAlphanumericPattern matches string that references to an alphanumeric issue, e.g. ABC-1234
-	IssueAlphanumericPattern = regexp.MustCompile(`( |^|\()[A-Z]{1,10}-[1-9][0-9]*\b`)
+	IssueAlphanumericPattern = regexp.MustCompile(`( |^|\(|\[)[A-Z]{1,10}-[1-9][0-9]*\b`)
 	// CrossReferenceIssueNumericPattern matches string that references a numeric issue in a different repository
 	// e.g. gogits/gogs#12345
 	CrossReferenceIssueNumericPattern = regexp.MustCompile(`( |^)[0-9a-zA-Z-_\.]+/[0-9a-zA-Z-_\.]+#[0-9]+\b`)

modules/markup/html_test.go

@@ -345,6 +345,7 @@ func TestRegExp_IssueNumericPattern(t *testing.T) {
 		"#1234",
 		"#0",
 		"#1234567890987654321",
+		"[#1234]",
 	}
 	falseTestCases := []string{
 		"# 1234",
@@ -355,6 +356,8 @@ func TestRegExp_IssueNumericPattern(t *testing.T) {
 		"#1A2B",
 		"",
 		"ABC",
+		"[]",
+		"[x]",
 	}
 
 	for _, testCase := range trueTestCases {
@@ -371,6 +374,7 @@ func TestRegExp_IssueAlphanumericPattern(t *testing.T) {
 		"A-1",
 		"RC-80",
 		"ABCDEFGHIJ-1234567890987654321234567890",
+		"[JIRA-134]",
 	}
 	falseTestCases := []string{
 		"RC-08",
@@ -383,6 +387,7 @@ func TestRegExp_IssueAlphanumericPattern(t *testing.T) {
 		"ABC",
 		"GG-",
 		"rm-1",
+		"[]",
 	}
 
 	for _, testCase := range trueTestCases {

modules/public/dynamic.go

@@ -12,10 +12,5 @@ import (
 
 // Static implements the macaron static handler for serving assets.
 func Static(opts *Options) macaron.Handler {
-	return macaron.Static(
-		opts.Directory,
-		macaron.StaticOptions{
-			SkipLogging: opts.SkipLogging,
-		},
-	)
+	return opts.staticHandler(opts.Directory)
 }

modules/public/public.go

@@ -5,7 +5,13 @@
 package public
 
 import (
+	"encoding/base64"
+	"log"
+	"net/http"
 	"path"
+	"path/filepath"
+	"strings"
+	"time"
 
 	"code.gitea.io/gitea/modules/setting"
 	"gopkg.in/macaron.v1"
@@ -19,15 +25,135 @@ import (
 // Options represents the available options to configure the macaron handler.
 type Options struct {
 	Directory   string
+	IndexFile   string
 	SkipLogging bool
+	// if set to true, will enable caching. Expires header will also be set to
+	// expire after the defined time.
+	ExpiresAfter time.Duration
+	FileSystem   http.FileSystem
+	Prefix       string
 }
 
 // Custom implements the macaron static handler for serving custom assets.
 func Custom(opts *Options) macaron.Handler {
-	return macaron.Static(
-		path.Join(setting.CustomPath, "public"),
-		macaron.StaticOptions{
-			SkipLogging: opts.SkipLogging,
-		},
-	)
+	return opts.staticHandler(path.Join(setting.CustomPath, "public"))
+}
+
+// staticFileSystem implements http.FileSystem interface.
+type staticFileSystem struct {
+	dir *http.Dir
+}
+
+func newStaticFileSystem(directory string) staticFileSystem {
+	if !filepath.IsAbs(directory) {
+		directory = filepath.Join(macaron.Root, directory)
+	}
+	dir := http.Dir(directory)
+	return staticFileSystem{&dir}
+}
+
+func (fs staticFileSystem) Open(name string) (http.File, error) {
+	return fs.dir.Open(name)
+}
+
+// StaticHandler sets up a new middleware for serving static files in the
+func StaticHandler(dir string, opts *Options) macaron.Handler {
+	return opts.staticHandler(dir)
+}
+
+func (opts *Options) staticHandler(dir string) macaron.Handler {
+	// Defaults
+	if len(opts.IndexFile) == 0 {
+		opts.IndexFile = "index.html"
+	}
+	// Normalize the prefix if provided
+	if opts.Prefix != "" {
+		// Ensure we have a leading '/'
+		if opts.Prefix[0] != '/' {
+			opts.Prefix = "/" + opts.Prefix
+		}
+		// Remove any trailing '/'
+		opts.Prefix = strings.TrimRight(opts.Prefix, "/")
+	}
+	if opts.FileSystem == nil {
+		opts.FileSystem = newStaticFileSystem(dir)
+	}
+
+	return func(ctx *macaron.Context, log *log.Logger) {
+		opts.handle(ctx, log, opts)
+	}
+}
+
+func (opts *Options) handle(ctx *macaron.Context, log *log.Logger, opt *Options) bool {
+	if ctx.Req.Method != "GET" && ctx.Req.Method != "HEAD" {
+		return false
+	}
+
+	file := ctx.Req.URL.Path
+	// if we have a prefix, filter requests by stripping the prefix
+	if opt.Prefix != "" {
+		if !strings.HasPrefix(file, opt.Prefix) {
+			return false
+		}
+		file = file[len(opt.Prefix):]
+		if file != "" && file[0] != '/' {
+			return false
+		}
+	}
+
+	f, err := opt.FileSystem.Open(file)
+	if err != nil {
+		return false
+	}
+	defer f.Close()
+
+	fi, err := f.Stat()
+	if err != nil {
+		log.Printf("[Static] %q exists, but fails to open: %v", file, err)
+		return true
+	}
+
+	// Try to serve index file
+	if fi.IsDir() {
+		// Redirect if missing trailing slash.
+		if !strings.HasSuffix(ctx.Req.URL.Path, "/") {
+			http.Redirect(ctx.Resp, ctx.Req.Request, ctx.Req.URL.Path+"/", http.StatusFound)
+			return true
+		}
+
+		f, err = opt.FileSystem.Open(file)
+		if err != nil {
+			return false // Discard error.
+		}
+		defer f.Close()
+
+		fi, err = f.Stat()
+		if err != nil || fi.IsDir() {
+			return true
+		}
+	}
+
+	if !opt.SkipLogging {
+		log.Println("[Static] Serving " + file)
+	}
+
+	// Add an Expires header to the static content
+	if opt.ExpiresAfter > 0 {
+		ctx.Resp.Header().Set("Expires", time.Now().Add(opt.ExpiresAfter).UTC().Format(http.TimeFormat))
+		tag := GenerateETag(string(fi.Size()), fi.Name(), fi.ModTime().UTC().Format(http.TimeFormat))
+		ctx.Resp.Header().Set("ETag", tag)
+		if ctx.Req.Header.Get("If-None-Match") == tag {
+			ctx.Resp.WriteHeader(304)
+			return false
+		}
+	}
+
+	http.ServeContent(ctx.Resp, ctx.Req.Request, file, fi.ModTime(), f)
+	return true
+}
+
+// GenerateETag generates an ETag based on size, filename and file modification time
+func GenerateETag(fileSize, fileName, modTime string) string {
+	etag := fileSize + fileName + modTime
+	return base64.StdEncoding.EncodeToString([]byte(etag))
 }

modules/public/static.go

@@ -13,17 +13,14 @@ import (
 
 // Static implements the macaron static handler for serving assets.
 func Static(opts *Options) macaron.Handler {
-	return macaron.Static(
-		opts.Directory,
-		macaron.StaticOptions{
-			SkipLogging: opts.SkipLogging,
-			FileSystem: bindata.Static(bindata.Options{
-				Asset:      Asset,
-				AssetDir:   AssetDir,
-				AssetInfo:  AssetInfo,
-				AssetNames: AssetNames,
-				Prefix:     "",
-			}),
-		},
-	)
+	opts.FileSystem = bindata.Static(bindata.Options{
+		Asset:      Asset,
+		AssetDir:   AssetDir,
+		AssetInfo:  AssetInfo,
+		AssetNames: AssetNames,
+		Prefix:     "",
+	})
+	// we don't need to pass the directory, because the directory var is only
+	// used when in the options there is no FileSystem.
+	return opts.staticHandler("")
 }

public/js/index.js

@@ -571,6 +571,7 @@ function initRepository() {
                 $editContentZone.html($('#edit-content-form').html());
                 $textarea = $segment.find('textarea');
                 issuesTribute.attach($textarea.get());
+                emojiTribute.attach($textarea.get());
 
                 // Give new write/preview data-tab name to distinguish from others
                 var $editContentForm = $editContentZone.find('.ui.comment.form');

public/less/_repository.less

@@ -1491,20 +1491,24 @@
         .desc {
             padding-top: 5px;
             color: #999;
-	    .progress-bar {
-		width: 80px;
-		height: 6px;
-		display: inline-block;
-		background-color: #eee;
-		overflow: hidden;
-		border-radius: 3px;
-		vertical-align: middle !important;
-	        .progress {
-	            background-color: #ccc;
-	            display: block;
-	    	    height: 100%;
-		}
-	    }
+            .checklist {
+                padding-left: 5px;
+                .progress-bar {
+                    margin-left: 2px;
+                    width: 80px;
+                    height: 6px;
+                    display: inline-block;
+                    background-color: #eee;
+                    overflow: hidden;
+                    border-radius: 3px;
+                    vertical-align: 2px !important;
+                    .progress {
+                        background-color: #ccc;
+                        display: block;
+                        height: 100%;
+                    }
+                }
+            }
             a.milestone {
                 padding-left: 5px;
                 color: #999!important;

public/less/_tribute.less

@@ -0,0 +1,26 @@
+.tribute-container {
+	box-shadow: 0px 1px 3px 1px #c7c7c7;
+	ul {
+		background: #ffffff;
+	}
+	li {
+		padding: 8px 12px;
+		border-bottom: 1px solid #dcdcdc;
+		img {
+			display: inline-block;
+			vertical-align: middle;
+			width: 28px;
+			height: 28px;
+			margin-right: 5px;
+		}
+		span.fullname {
+			font-weight: normal;
+			font-size: 0.8rem;
+			margin-left: 3px;
+		}
+	}
+	li.highlight, li:hover {
+		background: #2185D0;
+		color: #ffffff;
+	}
+}

public/less/index.less

@@ -1,3 +1,4 @@
+@import "_tribute";
 @import "_emojify";
 @import "_base";
 @import "_markdown";

public/vendor/plugins/tribute/tribute.css

@@ -7,33 +7,20 @@
   max-width: 500px;
   overflow: auto;
   display: block;
-  box-shadow: 0px 1px 3px 1px #c7c7c7;
   z-index: 999999; }
   .tribute-container ul {
     margin: 0;
     margin-top: 2px;
     padding: 0;
     list-style: none;
-    background: #ffffff; }
+    background: #efefef; }
   .tribute-container li {
-    padding: 8px 12px;
-    border-bottom: 1px solid #dcdcdc;
+    padding: 5px 5px;
     cursor: pointer; }
     .tribute-container li.highlight, .tribute-container li:hover {
-      background: #2185D0;
-      color: #ffffff;}
-    .tribute-container li img {
-      display: inline-block;
-      vertical-align: middle;
-      width: 28px;
-      margin-right: 5px;
-    }
+      background: #ddd; }
     .tribute-container li span {
       font-weight: bold; }
-    .tribute-container li span.fullname {
-      font-weight: normal;
-      font-size: 0.8rem;
-      margin-left: 3px;}
     .tribute-container li.no-match {
       cursor: default; }
   .tribute-container .menu-highlighted {

routers/repo/wiki.go

@@ -128,6 +128,9 @@ func renderWikiPage(ctx *context.Context, isViewPage bool) (*git.Repository, *gi
 			}
 			wikiName, err := models.WikiFilenameToName(entry.Name())
 			if err != nil {
+				if models.IsErrWikiInvalidFileName(err) {
+					continue
+				}
 				ctx.ServerError("WikiFilenameToName", err)
 				return nil, nil
 			} else if wikiName == "_Sidebar" || wikiName == "_Footer" {
@@ -262,6 +265,9 @@ func WikiPages(ctx *context.Context) {
 		}
 		wikiName, err := models.WikiFilenameToName(entry.Name())
 		if err != nil {
+			if models.IsErrWikiInvalidFileName(err) {
+				continue
+			}
 			ctx.ServerError("WikiFilenameToName", err)
 			return
 		}

routers/routes/routes.go

@@ -7,6 +7,7 @@ package routes
 import (
 	"os"
 	"path"
+	"time"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/auth"
@@ -53,21 +54,23 @@ func NewMacaron() *macaron.Macaron {
 	}
 	m.Use(public.Custom(
 		&public.Options{
-			SkipLogging: setting.DisableRouterLog,
+			SkipLogging:  setting.DisableRouterLog,
+			ExpiresAfter: time.Hour * 6,
 		},
 	))
 	m.Use(public.Static(
 		&public.Options{
-			Directory:   path.Join(setting.StaticRootPath, "public"),
-			SkipLogging: setting.DisableRouterLog,
+			Directory:    path.Join(setting.StaticRootPath, "public"),
+			SkipLogging:  setting.DisableRouterLog,
+			ExpiresAfter: time.Hour * 6,
 		},
 	))
-	m.Use(macaron.Static(
+	m.Use(public.StaticHandler(
 		setting.AvatarUploadPath,
-		macaron.StaticOptions{
-			Prefix:      "avatars",
-			SkipLogging: setting.DisableRouterLog,
-			ETag:        true,
+		&public.Options{
+			Prefix:       "avatars",
+			SkipLogging:  setting.DisableRouterLog,
+			ExpiresAfter: time.Hour * 6,
 		},
 	))
 

templates/base/footer.tmpl

@@ -89,6 +89,38 @@
 		issuesTribute.attach(document.getElementById('content'))
 	</script>
 	{{end}}
+	<script>
+		var emojiTribute = new Tribute({
+			collection: [{
+				trigger: ':',
+				requireLeadingSpace: true,
+				values: function (text, cb) {
+					var array = emojify.emojiNames;
+					var data = [];
+					for(var j=0; j<array.length; j++) {
+						if(array[j].indexOf(text) !== -1) {
+							data.push(array[j]);
+							if(data.length > 5) {
+								break;
+							}
+						}
+					}
+					cb(data);
+				},
+				lookup: function (item) {
+					return item;
+				},
+				selectTemplate: function (item) {
+					if (typeof item === 'undefinied') return null;
+					return ':' + item.original + ':';
+				},
+				menuItemTemplate: function (item) {
+					return '<img class="emoji" src="{{AppSubUrl}}/vendor/plugins/emojify/images/' + item.original + '.png"/>' + item.original;
+				}
+			}]
+		});
+		emojiTribute.attach(document.getElementById('content'))
+	</script>
 {{end}}
 	<script src="{{AppSubUrl}}/vendor/plugins/autolink/autolink.js"></script>
 	<script src="{{AppSubUrl}}/vendor/plugins/emojify/emojify.min.js"></script>

templates/repo/issue/list.tmpl

@@ -203,7 +203,9 @@
 						{{$tasks := .GetTasks}}
 						{{if gt $tasks 0}}
 							{{$tasksDone := .GetTasksDone}}
-							<span class="octicon octicon-checklist"></span> {{$tasksDone}} / {{$tasks}} <span class="progress-bar"><span class="progress" style="width:calc(100% * {{$tasksDone}} / {{$tasks}});"></span></span>
+							<span class="checklist">
+								<span class="octicon octicon-checklist"></span> {{$tasksDone}} / {{$tasks}} <span class="progress-bar"><span class="progress" style="width:calc(100% * {{$tasksDone}} / {{$tasks}});"></span></span>
+							</span>
 						{{end}}
 						{{if .Milestone}}
 							<a class="milestone" href="{{$.Link}}?q={{$.Keyword}}&type={{$.ViewType}}&state={{$.State}}&labels={{$.SelectLabels}}&milestone={{.Milestone.ID}}&assignee={{$.AssigneeID}}">

templates/user/dashboard/issues.tmpl

@@ -90,7 +90,9 @@
 								{{$tasks := .GetTasks}}
 								{{if gt $tasks 0}}
 									{{$tasksDone := .GetTasksDone}}
-									<span class="octicon octicon-checklist"></span> {{$tasksDone}} / {{$tasks}} <span class="progress-bar"><span class="progress" style="width:calc(100% * {{$tasksDone}} / {{$tasks}});"></span></span>
+									<span class="checklist">
+										<span class="octicon octicon-checklist"></span> {{$tasksDone}} / {{$tasks}} <span class="progress-bar"><span class="progress" style="width:calc(100% * {{$tasksDone}} / {{$tasks}});"></span></span>
+									</span>
 								{{end}}
 							</p>
 						</li>

vendor/github.com/blevesearch/bleve/analysis/token/unique/unique.go

@@ -0,0 +1,53 @@
+//  Copyright (c) 2018 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package unique
+
+import (
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/registry"
+)
+
+const Name = "unique"
+
+// UniqueTermFilter retains only the tokens which mark the first occurence of
+// a term. Tokens whose term appears in a preceding token are dropped.
+type UniqueTermFilter struct{}
+
+func NewUniqueTermFilter() *UniqueTermFilter {
+	return &UniqueTermFilter{}
+}
+
+func (f *UniqueTermFilter) Filter(input analysis.TokenStream) analysis.TokenStream {
+	encounteredTerms := make(map[string]struct{}, len(input)/4)
+	j := 0
+	for _, token := range input {
+		term := string(token.Term)
+		if _, ok := encounteredTerms[term]; ok {
+			continue
+		}
+		encounteredTerms[term] = struct{}{}
+		input[j] = token
+		j++
+	}
+	return input[:j]
+}
+
+func UniqueTermFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {
+	return NewUniqueTermFilter(), nil
+}
+
+func init() {
+	registry.RegisterTokenFilter(Name, UniqueTermFilterConstructor)
+}

vendor/github.com/ethantkoenig/rupture/Gopkg.lock

@@ -0,0 +1,173 @@
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[[projects]]
+  name = "github.com/RoaringBitmap/roaring"
+  packages = ["."]
+  revision = "84551f0e309d6f9bafa428ef39b31ab7f16ff7b8"
+  version = "v0.4.1"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/Smerity/govarint"
+  packages = ["."]
+  revision = "7265e41f48f15fd61751e16da866af3c704bb3ab"
+
+[[projects]]
+  name = "github.com/blevesearch/bleve"
+  packages = [
+    ".",
+    "analysis",
+    "analysis/analyzer/standard",
+    "analysis/datetime/flexible",
+    "analysis/datetime/optional",
+    "analysis/lang/en",
+    "analysis/token/lowercase",
+    "analysis/token/porter",
+    "analysis/token/stop",
+    "analysis/tokenizer/unicode",
+    "document",
+    "geo",
+    "index",
+    "index/scorch",
+    "index/scorch/mergeplan",
+    "index/scorch/segment",
+    "index/scorch/segment/mem",
+    "index/scorch/segment/zap",
+    "index/store",
+    "index/store/boltdb",
+    "index/store/gtreap",
+    "index/upsidedown",
+    "mapping",
+    "numeric",
+    "registry",
+    "search",
+    "search/collector",
+    "search/facet",
+    "search/highlight",
+    "search/highlight/format/html",
+    "search/highlight/fragmenter/simple",
+    "search/highlight/highlighter/html",
+    "search/highlight/highlighter/simple",
+    "search/query",
+    "search/scorer",
+    "search/searcher"
+  ]
+  revision = "a3b125508b4443344b596888ca58467b6c9310b9"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/blevesearch/go-porterstemmer"
+  packages = ["."]
+  revision = "23a2c8e5cf1f380f27722c6d2ae8896431dc7d0e"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/blevesearch/segment"
+  packages = ["."]
+  revision = "762005e7a34fd909a84586299f1dd457371d36ee"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/boltdb/bolt"
+  packages = ["."]
+  revision = "9da31745363232bc1e27dbab3569e77383a51585"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/couchbase/vellum"
+  packages = [
+    ".",
+    "regexp",
+    "utf8"
+  ]
+  revision = "ed84a675e24ed0a0bf6859b1ddec7e7c858354bd"
+
+[[projects]]
+  name = "github.com/davecgh/go-spew"
+  packages = ["spew"]
+  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
+  version = "v1.1.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/edsrzf/mmap-go"
+  packages = ["."]
+  revision = "0bce6a6887123b67a60366d2c9fe2dfb74289d2e"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/glycerine/go-unsnap-stream"
+  packages = ["."]
+  revision = "62a9a9eb44fd8932157b1a8ace2149eff5971af6"
+
+[[projects]]
+  name = "github.com/golang/protobuf"
+  packages = ["proto"]
+  revision = "925541529c1fa6821df4e44ce2723319eb2be768"
+  version = "v1.0.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/golang/snappy"
+  packages = ["."]
+  revision = "553a641470496b2327abcac10b36396bd98e45c9"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/mschoch/smat"
+  packages = ["."]
+  revision = "90eadee771aeab36e8bf796039b8c261bebebe4f"
+
+[[projects]]
+  name = "github.com/philhofer/fwd"
+  packages = ["."]
+  revision = "bb6d471dc95d4fe11e432687f8b70ff496cf3136"
+  version = "v1.0.0"
+
+[[projects]]
+  name = "github.com/pmezard/go-difflib"
+  packages = ["difflib"]
+  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
+  version = "v1.0.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/steveyen/gtreap"
+  packages = ["."]
+  revision = "0abe01ef9be25c4aedc174758ec2d917314d6d70"
+
+[[projects]]
+  name = "github.com/stretchr/testify"
+  packages = ["assert"]
+  revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71"
+  version = "v1.2.1"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/tinylib/msgp"
+  packages = ["msgp"]
+  revision = "03a79185462ad029a6e7e05b2f3f3e0498d0a6c0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/willf/bitset"
+  packages = ["."]
+  revision = "1a37ad96e8c1a11b20900a232874843b5174221f"
+
+[[projects]]
+  name = "golang.org/x/net"
+  packages = ["context"]
+  revision = "309822c5b9b9f80db67f016069a12628d94fad34"
+
+[[projects]]
+  name = "golang.org/x/sys"
+  packages = ["unix"]
+  revision = "3dbebcf8efb6a5011a60c2b4591c1022a759af8a"
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  inputs-digest = "61c759f0c1136cadf86ae8a30bb78edf33fc844cdcb2316469b4ae14a8d051b0"
+  solver-name = "gps-cdcl"
+  solver-version = 1

vendor/github.com/ethantkoenig/rupture/Gopkg.toml

@@ -0,0 +1,34 @@
+# Gopkg.toml example
+#
+# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+#   name = "github.com/user/project"
+#   version = "1.0.0"
+#
+# [[constraint]]
+#   name = "github.com/user/project2"
+#   branch = "dev"
+#   source = "github.com/myfork/project2"
+#
+# [[override]]
+#   name = "github.com/x/y"
+#   version = "2.4.0"
+#
+# [prune]
+#   non-go = false
+#   go-tests = true
+#   unused-packages = true
+
+
+[[constraint]]
+  name = "github.com/stretchr/testify"
+  version = "1.2.1"
+
+[prune]
+  go-tests = true
+  unused-packages = true

vendor/github.com/ethantkoenig/rupture/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Ethan Koenig
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/ethantkoenig/rupture/README.md

@@ -0,0 +1,13 @@
+# rupture
+
+[![Build Status](https://travis-ci.org/ethantkoenig/rupture.svg?branch=master)](https://travis-ci.org/ethantkoenig/rupture) [![GoDoc](https://godoc.org/github.com/ethantkoenig/rupture?status.svg)](https://godoc.org/github.com/ethantkoenig/rupture) [![Go Report Card](https://goreportcard.com/badge/blevesearch/bleve)](https://goreportcard.com/report/blevesearch/bleve)
+
+An explosive companion to the [bleve indexing library](https://www.github.com/blevesearch/bleve)
+
+## Features
+
+`rupture` includes the following additions to `bleve`:
+
+- __Flushing batches__: Batches of operation which automatically flush to the underlying bleve index.
+- __Sharded indices__: An index-like abstraction built on top of several underlying indices. Sharded indices provide lower write latencies for indices with large amounts of data.
+- __Index metadata__: Track index version for easily managing migrations and schema changes.

vendor/github.com/ethantkoenig/rupture/flushing_batch.go

@@ -0,0 +1,67 @@
+package rupture
+
+import (
+	"github.com/blevesearch/bleve"
+)
+
+// FlushingBatch is a batch of operations that automatically flushes to the
+// underlying index once it reaches a certain size.
+type FlushingBatch interface {
+	// Index adds the specified index operation batch, possibly triggering a
+	// flush.
+	Index(id string, data interface{}) error
+	// Remove adds the specified delete operation to the batch, possibly
+	// triggering a flush.
+	Delete(id string) error
+	// Flush flushes the batch's contents.
+	Flush() error
+}
+
+type singleIndexFlushingBatch struct {
+	maxBatchSize int
+	batch        *bleve.Batch
+	index        bleve.Index
+}
+
+func newFlushingBatch(index bleve.Index, maxBatchSize int) *singleIndexFlushingBatch {
+	return &singleIndexFlushingBatch{
+		maxBatchSize: maxBatchSize,
+		batch:        index.NewBatch(),
+		index:        index,
+	}
+}
+
+// NewFlushingBatch creates a new flushing batch for the specified index. Once
+// the number of operations in the batch reaches the specified limit, the batch
+// automatically flushes its operations to the index.
+func NewFlushingBatch(index bleve.Index, maxBatchSize int) FlushingBatch {
+	return newFlushingBatch(index, maxBatchSize)
+}
+
+func (b *singleIndexFlushingBatch) Index(id string, data interface{}) error {
+	if err := b.batch.Index(id, data); err != nil {
+		return err
+	}
+	return b.flushIfFull()
+}
+
+func (b *singleIndexFlushingBatch) Delete(id string) error {
+	b.batch.Delete(id)
+	return b.flushIfFull()
+}
+
+func (b *singleIndexFlushingBatch) flushIfFull() error {
+	if b.batch.Size() < b.maxBatchSize {
+		return nil
+	}
+	return b.Flush()
+}
+
+func (b *singleIndexFlushingBatch) Flush() error {
+	err := b.index.Batch(b.batch)
+	if err != nil {
+		return err
+	}
+	b.batch.Reset()
+	return nil
+}

vendor/github.com/ethantkoenig/rupture/metadata.go

@@ -0,0 +1,68 @@
+package rupture
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+const metaFilename = "rupture_meta.json"
+
+func indexMetadataPath(dir string) string {
+	return filepath.Join(dir, metaFilename)
+}
+
+// IndexMetadata contains metadata about a bleve index.
+type IndexMetadata struct {
+	// The version of the data in the index. This can be useful for tracking
+	// schema changes or data migrations.
+	Version int `json:"version"`
+}
+
+// in addition to the user-exposed metadata, we keep additional, internal-only
+// metadata for sharded indices.
+const shardedMetadataFilename = "rupture_sharded_meta.json"
+
+func shardedIndexMetadataPath(dir string) string {
+	return filepath.Join(dir, shardedMetadataFilename)
+}
+
+type shardedIndexMetadata struct {
+	NumShards int `json:"num_shards"`
+}
+
+func readJSON(path string, meta interface{}) error {
+	metaBytes, err := ioutil.ReadFile(path)
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(metaBytes, meta)
+}
+
+func writeJSON(path string, meta interface{}) error {
+	metaBytes, err := json.Marshal(meta)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(path, metaBytes, 0666)
+}
+
+// ReadIndexMetadata returns the metadata for the index at the specified path.
+// If no such index metadata exists, an empty metadata and a nil error are
+// returned.
+func ReadIndexMetadata(path string) (*IndexMetadata, error) {
+	meta := &IndexMetadata{}
+	metaPath := indexMetadataPath(path)
+	if _, err := os.Stat(metaPath); os.IsNotExist(err) {
+		return meta, nil
+	} else if err != nil {
+		return nil, err
+	}
+	return meta, readJSON(metaPath, meta)
+}
+
+// WriteIndexMetadata writes metadata for the index at the specified path.
+func WriteIndexMetadata(path string, meta *IndexMetadata) error {
+	return writeJSON(indexMetadataPath(path), meta)
+}

vendor/github.com/ethantkoenig/rupture/sharded_index.go

@@ -0,0 +1,146 @@
+package rupture
+
+import (
+	"fmt"
+	"hash/fnv"
+	"path/filepath"
+	"strconv"
+
+	"github.com/blevesearch/bleve"
+	"github.com/blevesearch/bleve/document"
+	"github.com/blevesearch/bleve/mapping"
+)
+
+// ShardedIndex an index that is built onto of multiple underlying bleve
+// indices (i.e. shards). Similar to bleve's index aliases, some methods may
+// not be supported.
+type ShardedIndex interface {
+	bleve.Index
+	shards() []bleve.Index
+}
+
+// a type alias for bleve.Index, so that the anonymous field of
+// shardedIndex does not conflict with the Index(..) method.
+type bleveIndex bleve.Index
+
+type shardedIndex struct {
+	bleveIndex
+	indices []bleve.Index
+}
+
+func hash(id string, n int) uint64 {
+	fnvHash := fnv.New64()
+	fnvHash.Write([]byte(id))
+	return fnvHash.Sum64() % uint64(n)
+}
+
+func childIndexerPath(rootPath string, i int) string {
+	return filepath.Join(rootPath, strconv.Itoa(i))
+}
+
+// NewShardedIndex creates a sharded index at the specified path, with the
+// specified mapping and number of shards.
+func NewShardedIndex(path string, mapping mapping.IndexMapping, numShards int) (ShardedIndex, error) {
+	if numShards <= 0 {
+		return nil, fmt.Errorf("Invalid number of shards: %d", numShards)
+	}
+	err := writeJSON(shardedIndexMetadataPath(path), &shardedIndexMetadata{NumShards: numShards})
+	if err != nil {
+		return nil, err
+	}
+
+	s := &shardedIndex{
+		indices: make([]bleve.Index, numShards),
+	}
+	for i := 0; i < numShards; i++ {
+		s.indices[i], err = bleve.New(childIndexerPath(path, i), mapping)
+		if err != nil {
+			return nil, err
+		}
+	}
+	s.bleveIndex = bleve.NewIndexAlias(s.indices...)
+	return s, nil
+}
+
+// OpenShardedIndex opens a sharded index at the specified path.
+func OpenShardedIndex(path string) (ShardedIndex, error) {
+	var meta shardedIndexMetadata
+	var err error
+	if err = readJSON(shardedIndexMetadataPath(path), &meta); err != nil {
+		return nil, err
+	}
+
+	s := &shardedIndex{
+		indices: make([]bleve.Index, meta.NumShards),
+	}
+	for i := 0; i < meta.NumShards; i++ {
+		s.indices[i], err = bleve.Open(childIndexerPath(path, i))
+		if err != nil {
+			return nil, err
+		}
+	}
+	s.bleveIndex = bleve.NewIndexAlias(s.indices...)
+	return s, nil
+}
+
+func (s *shardedIndex) Index(id string, data interface{}) error {
+	return s.indices[hash(id, len(s.indices))].Index(id, data)
+}
+
+func (s *shardedIndex) Delete(id string) error {
+	return s.indices[hash(id, len(s.indices))].Delete(id)
+}
+
+func (s *shardedIndex) Document(id string) (*document.Document, error) {
+	return s.indices[hash(id, len(s.indices))].Document(id)
+}
+
+func (s *shardedIndex) Close() error {
+	if err := s.bleveIndex.Close(); err != nil {
+		return err
+	}
+	for _, index := range s.indices {
+		if err := index.Close(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *shardedIndex) shards() []bleve.Index {
+	return s.indices
+}
+
+type shardedIndexFlushingBatch struct {
+	batches []*singleIndexFlushingBatch
+}
+
+// NewShardedFlushingBatch creates a flushing batch with the specified batch
+// size for the specified sharded index.
+func NewShardedFlushingBatch(index ShardedIndex, maxBatchSize int) FlushingBatch {
+	indices := index.shards()
+	b := &shardedIndexFlushingBatch{
+		batches: make([]*singleIndexFlushingBatch, len(indices)),
+	}
+	for i, index := range indices {
+		b.batches[i] = newFlushingBatch(index, maxBatchSize)
+	}
+	return b
+}
+
+func (b *shardedIndexFlushingBatch) Index(id string, data interface{}) error {
+	return b.batches[hash(id, len(b.batches))].Index(id, data)
+}
+
+func (b *shardedIndexFlushingBatch) Delete(id string) error {
+	return b.batches[hash(id, len(b.batches))].Delete(id)
+}
+
+func (b *shardedIndexFlushingBatch) Flush() error {
+	for _, batch := range b.batches {
+		if err := batch.Flush(); err != nil {
+			return err
+		}
+	}
+	return nil
+}

vendor/vendor.json

@@ -128,6 +128,12 @@
 			"revision": "174f8ed44a0bf65e7c8fb228b60b58de62654cd2",
 			"revisionTime": "2017-06-28T17:18:15Z"
 		},
+		{
+			"checksumSHA1": "unacAFTLwgpg7wyI/mYf7Zd9eaU=",
+			"path": "github.com/blevesearch/bleve/analysis/token/unique",
+			"revision": "ff210fbc6d348ad67aa5754eaea11a463fcddafd",
+			"revisionTime": "2018-02-01T18:20:06Z"
+		},
 		{
 			"checksumSHA1": "q7C04nlJLxKmemXLop0oyJhfi5M=",
 			"path": "github.com/blevesearch/bleve/analysis/tokenizer/unicode",
@@ -347,6 +353,12 @@
 			"revision": "57eb5e1fc594ad4b0b1dbea7b286d299e0cb43c2",
 			"revisionTime": "2015-12-24T04:54:52Z"
 		},
+		{
+			"checksumSHA1": "06ofBxeJ9c4LS2p31PCMIj7IjJU=",
+			"path": "github.com/ethantkoenig/rupture",
+			"revision": "0a76f03a811abcca2e6357329b673e9bb8ef9643",
+			"revisionTime": "2018-02-03T18:25:44Z"
+		},
 		{
 			"checksumSHA1": "imR2wF388/0fBU6RRWx8RvTi8Q8=",
 			"path": "github.com/facebookgo/clock",