peter/go-gitea
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Sync releases table with tags on push and for mirrors (#2459) (#2554)

Browse Source 
* Sync releases table with tags on push and for mirrors

* Code style fixes

* Fix api to return only releases

* Optimize release creation and update
Minimize posibility of race conditions

* Fix release lower tag name updating

* handle tag reference update by addionally comparing commit id
This commit is contained in:
Lauris BH 2017-09-21 01:08:22 +03:00 committed by GitHub
parent 25e71ad41e
commit e232c49b10
10 changed files with 370 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
models/migrations/migrations.go
View File

@ -128,6 +128,8 @@ var migrations = []Migration{
NewMigration("remove commits and settings unit types", removeCommitsUnitType),
// v43 -> v44
NewMigration("fix protected branch can push value to false", fixProtectedBranchCanPushValue),
// v42 -> v43
NewMigration("add tags to releases and sync existing repositories", releaseAddColumnIsTagAndSyncTags),
}
// Migrate database to current version

57
models/migrations/v42.go Normal file
View File

@ -0,0 +1,57 @@
// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"fmt"
"code.gitea.io/git"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
"github.com/go-xorm/xorm"
)
// ReleaseV39 describes the added field for Release
type ReleaseV39 struct {
IsTag bool `xorm:"NOT NULL DEFAULT false"`
}
// TableName will be invoked by XORM to customrize the table name
func (*ReleaseV39) TableName() string {
return "release"
}
func releaseAddColumnIsTagAndSyncTags(x *xorm.Engine) error {
if err := x.Sync2(new(ReleaseV39)); err != nil {
return fmt.Errorf("Sync2: %v", err)
}
// For the sake of SQLite3, we can't use x.Iterate here.
offset := 0
pageSize := 20
for {
repos := make([]*models.Repository, 0, pageSize)
if err := x.Table("repository").Asc("id").Limit(pageSize, offset).Find(&repos); err != nil {
return fmt.Errorf("select repos [offset: %d]: %v", offset, err)
}
for _, repo := range repos {
gitRepo, err := git.OpenRepository(repo.RepoPath())
if err != nil {
log.Warn("OpenRepository: %v", err)
continue
}
if err = models.SyncReleasesWithTags(repo, gitRepo); err != nil {
log.Warn("SyncReleasesWithTags: %v", err)
}
}
if len(repos) < pageSize {
break
}
offset += pageSize
}
return nil
}

69
models/release.go
View File

@ -34,7 +34,8 @@ type Release struct {
NumCommitsBehind int64 `xorm:"-"`
Note string `xorm:"TEXT"`
IsDraft bool `xorm:"NOT NULL DEFAULT false"`
IsPrerelease bool
IsPrerelease bool `xorm:"NOT NULL DEFAULT false"`
IsTag bool `xorm:"NOT NULL DEFAULT false"`
Attachments []*Attachment `xorm:"-"`
@ -139,19 +140,20 @@ func createTag(gitRepo *git.Repository, rel *Release) error {
}
return err
}
} else {
rel.LowerTagName = strings.ToLower(rel.TagName)
}
commit, err := gitRepo.GetTagCommit(rel.TagName)
if err != nil {
return fmt.Errorf("GetTagCommit: %v", err)
}
rel.Sha1 = commit.ID.String()
rel.CreatedUnix = commit.Author.When.Unix()
rel.NumCommits, err = commit.CommitsCount()
if err != nil {
return fmt.Errorf("CommitsCount: %v", err)
}
}
}
return nil
}
@ -236,6 +238,7 @@ func GetReleaseByID(id int64) (*Release, error) {
// FindReleasesOptions describes the conditions to Find releases
type FindReleasesOptions struct {
IncludeDrafts bool
IncludeTags bool
TagNames []string
}
@ -246,6 +249,9 @@ func (opts *FindReleasesOptions) toConds(repoID int64) builder.Cond {
if !opts.IncludeDrafts {
cond = cond.And(builder.Eq{"is_draft": false})
}
if !opts.IncludeTags {
cond = cond.And(builder.Eq{"is_tag": false})
}
if len(opts.TagNames) > 0 {
cond = cond.And(builder.In("tag_name", opts.TagNames))
}
@ -361,6 +367,8 @@ func UpdateRelease(gitRepo *git.Repository, rel *Release, attachmentUUIDs []stri
if err = createTag(gitRepo, rel); err != nil {
return err
}
rel.LowerTagName = strings.ToLower(rel.TagName)
_, err = x.Id(rel.ID).AllCols().Update(rel)
if err != nil {
return err
@ -397,11 +405,64 @@ func DeleteReleaseByID(id int64, u *User, delTag bool) error {
if err != nil && !strings.Contains(stderr, "not found") {
return fmt.Errorf("git tag -d: %v - %s", err, stderr)
}
}
if _, err = x.Id(rel.ID).Delete(new(Release)); err != nil {
return fmt.Errorf("Delete: %v", err)
}
} else {
rel.IsTag = true
rel.IsDraft = false
rel.IsPrerelease = false
rel.Title = ""
rel.Note = ""
if _, err = x.Id(rel.ID).AllCols().Update(rel); err != nil {
return fmt.Errorf("Update: %v", err)
}
}
return nil
}
// SyncReleasesWithTags synchronizes release table with repository tags
func SyncReleasesWithTags(repo *Repository, gitRepo *git.Repository) error {
existingRelTags := make(map[string]struct{})
opts := FindReleasesOptions{IncludeDrafts: true, IncludeTags: true}
for page := 1; ; page++ {
rels, err := GetReleasesByRepoID(repo.ID, opts, page, 100)
if err != nil {
return fmt.Errorf("GetReleasesByRepoID: %v", err)
}
if len(rels) == 0 {
break
}
for _, rel := range rels {
if rel.IsDraft {
continue
}
commitID, err := gitRepo.GetTagCommitID(rel.TagName)
if err != nil {
return fmt.Errorf("GetTagCommitID: %v", err)
}
if !gitRepo.IsTagExist(rel.TagName) || commitID != rel.Sha1 {
if err := pushUpdateDeleteTag(repo, gitRepo, rel.TagName); err != nil {
return fmt.Errorf("pushUpdateDeleteTag: %v", err)
}
} else {
existingRelTags[strings.ToLower(rel.TagName)] = struct{}{}
}
}
}
tags, err := gitRepo.GetTags()
if err != nil {
return fmt.Errorf("GetTags: %v", err)
}
for _, tagName := range tags {
if _, ok := existingRelTags[strings.ToLower(tagName)]; !ok {
if err := pushUpdateAddTag(repo, gitRepo, tagName); err != nil {
return fmt.Errorf("pushUpdateAddTag: %v", err)
}
}
}
return nil
}

4
models/repo.go
View File

@ -912,6 +912,10 @@ func MigrateRepository(u *User, opts MigrateRepoOptions) (*Repository, error) {
if headBranch != nil {
repo.DefaultBranch = headBranch.Name
}
if err = SyncReleasesWithTags(repo, gitRepo); err != nil {
log.Error(4, "Failed to synchronize tags to releases for repository: %v", err)
}
}
if err = repo.UpdateSize(); err != nil {

10
models/repo_mirror.go
View File

@ -13,6 +13,7 @@ import (
"github.com/go-xorm/xorm"
"gopkg.in/ini.v1"
"code.gitea.io/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/process"
"code.gitea.io/gitea/modules/setting"
@ -148,6 +149,15 @@ func (m *Mirror) runSync() bool {
return false
}
gitRepo, err := git.OpenRepository(repoPath)
if err != nil {
log.Error(4, "OpenRepository: %v", err)
return false
}
if err = SyncReleasesWithTags(m.Repo, gitRepo); err != nil {
log.Error(4, "Failed to synchronize tags to releases for repository: %v", err)
}
if err := m.Repo.UpdateSize(); err != nil {
log.Error(4, "Failed to update size for mirror repository: %v", err)
}

107
models/update.go
View File

@ -81,6 +81,93 @@ func PushUpdate(branch string, opt PushUpdateOptions) error {
return nil
}
func pushUpdateDeleteTag(repo *Repository, gitRepo *git.Repository, tagName string) error {
rel, err := GetRelease(repo.ID, tagName)
if err != nil {
if IsErrReleaseNotExist(err) {
return nil
}
return fmt.Errorf("GetRelease: %v", err)
}
if rel.IsTag {
if _, err = x.Id(rel.ID).Delete(new(Release)); err != nil {
return fmt.Errorf("Delete: %v", err)
}
} else {
rel.IsDraft = true
rel.NumCommits = 0
rel.Sha1 = ""
if _, err = x.Id(rel.ID).AllCols().Update(rel); err != nil {
return fmt.Errorf("Update: %v", err)
}
}
return nil
}
func pushUpdateAddTag(repo *Repository, gitRepo *git.Repository, tagName string) error {
rel, err := GetRelease(repo.ID, tagName)
if err != nil && !IsErrReleaseNotExist(err) {
return fmt.Errorf("GetRelease: %v", err)
}
tag, err := gitRepo.GetTag(tagName)
if err != nil {
return fmt.Errorf("GetTag: %v", err)
}
commit, err := tag.Commit()
if err != nil {
return fmt.Errorf("Commit: %v", err)
}
tagCreatedUnix := commit.Author.When.Unix()
author, err := GetUserByEmail(commit.Author.Email)
if err != nil && !IsErrUserNotExist(err) {
return fmt.Errorf("GetUserByEmail: %v", err)
}
commitsCount, err := commit.CommitsCount()
if err != nil {
return fmt.Errorf("CommitsCount: %v", err)
}
if rel == nil {
rel = &Release{
RepoID: repo.ID,
Title: "",
TagName: tagName,
LowerTagName: strings.ToLower(tagName),
Target: "",
Sha1: commit.ID.String(),
NumCommits: commitsCount,
Note: "",
IsDraft: false,
IsPrerelease: false,
IsTag: true,
CreatedUnix: tagCreatedUnix,
}
if author != nil {
rel.PublisherID = author.ID
}
if _, err = x.InsertOne(rel); err != nil {
return fmt.Errorf("InsertOne: %v", err)
}
} else {
rel.Sha1 = commit.ID.String()
rel.CreatedUnix = tagCreatedUnix
rel.NumCommits = commitsCount
rel.IsDraft = false
if rel.IsTag && author != nil {
rel.PublisherID = author.ID
}
if _, err = x.Id(rel.ID).AllCols().Update(rel); err != nil {
return fmt.Errorf("Update: %v", err)
}
}
return nil
}
func pushUpdate(opts PushUpdateOptions) (repo *Repository, err error) {
isNewRef := opts.OldCommitID == git.EmptySHA
isDelRef := opts.NewCommitID == git.EmptySHA
@ -106,23 +193,31 @@ func pushUpdate(opts PushUpdateOptions) (repo *Repository, err error) {
return nil, fmt.Errorf("GetRepositoryByName: %v", err)
}
if isDelRef {
log.GitLogger.Info("Reference '%s' has been deleted from '%s/%s' by %s",
opts.RefFullName, opts.RepoUserName, opts.RepoName, opts.PusherName)
return repo, nil
}
gitRepo, err := git.OpenRepository(repoPath)
if err != nil {
return nil, fmt.Errorf("OpenRepository: %v", err)
}
if isDelRef {
// Tag has been deleted
if strings.HasPrefix(opts.RefFullName, git.TagPrefix) {
err = pushUpdateDeleteTag(repo, gitRepo, opts.RefFullName[len(git.TagPrefix):])
if err != nil {
return nil, fmt.Errorf("pushUpdateDeleteTag: %v", err)
}
}
log.GitLogger.Info("Reference '%s' has been deleted from '%s/%s' by %s",
opts.RefFullName, opts.RepoUserName, opts.RepoName, opts.PusherName)
return repo, nil
}
if err = repo.UpdateSize(); err != nil {
log.Error(4, "Failed to update size for repository: %v", err)
}
// Push tags.
if strings.HasPrefix(opts.RefFullName, git.TagPrefix) {
pushUpdateAddTag(repo, gitRepo, opts.RefFullName[len(git.TagPrefix):])
if err := CommitRepoAction(CommitRepoActionOptions{
PusherName: opts.PusherName,
RepoOwnerID: owner.ID,

1
modules/context/repo.go
View File

@ -347,6 +347,7 @@ func RepoAssignment() macaron.Handler {
count, err := models.GetReleaseCountByRepoID(ctx.Repo.Repository.ID, models.FindReleasesOptions{
IncludeDrafts: false,
IncludeTags: true,
})
if err != nil {
ctx.Handle(500, "GetReleaseCountByRepoID", err)

55
routers/api/v1/repo/release.go
View File

@ -5,8 +5,6 @@
package repo
import (
"strings"
api "code.gitea.io/sdk/gitea"
"code.gitea.io/gitea/models"
@ -36,6 +34,7 @@ func GetRelease(ctx *context.APIContext) {
func ListReleases(ctx *context.APIContext) {
releases, err := models.GetReleasesByRepoID(ctx.Repo.Repository.ID, models.FindReleasesOptions{
IncludeDrafts: ctx.Repo.AccessMode >= models.AccessModeWrite,
IncludeTags: false,
}, 1, 2147483647)
if err != nil {
ctx.Error(500, "GetReleasesByRepoID", err)
@ -62,35 +61,23 @@ func CreateRelease(ctx *context.APIContext, form api.CreateReleaseOption) {
ctx.Status(404)
return
}
tag, err := ctx.Repo.GitRepo.GetTag(form.TagName)
rel, err := models.GetRelease(ctx.Repo.Repository.ID, form.TagName)
if err != nil {
ctx.Error(500, "GetTag", err)
if !models.IsErrReleaseNotExist(err) {
ctx.Handle(500, "GetRelease", err)
return
}
commit, err := tag.Commit()
if err != nil {
ctx.Error(500, "Commit", err)
return
}
commitsCount, err := commit.CommitsCount()
if err != nil {
ctx.Error(500, "CommitsCount", err)
return
}
rel := &models.Release{
rel = &models.Release{
RepoID: ctx.Repo.Repository.ID,
PublisherID: ctx.User.ID,
Publisher: ctx.User,
TagName: form.TagName,
LowerTagName: strings.ToLower(form.TagName),
Target: form.Target,
Title: form.Title,
Sha1: commit.ID.String(),
NumCommits: commitsCount,
Note: form.Note,
IsDraft: form.IsDraft,
IsPrerelease: form.IsPrerelease,
CreatedUnix: commit.Author.When.Unix(),
IsTag: false,
}
if err := models.CreateRelease(ctx.Repo.GitRepo, rel, nil); err != nil {
if models.IsErrReleaseAlreadyExist(err) {
@ -100,6 +87,24 @@ func CreateRelease(ctx *context.APIContext, form api.CreateReleaseOption) {
}
return
}
} else {
if !rel.IsTag {
ctx.Status(409)
return
}
rel.Title = form.Title
rel.Note = form.Note
rel.IsDraft = form.IsDraft
rel.IsPrerelease = form.IsPrerelease
rel.PublisherID = ctx.User.ID
rel.IsTag = false
if err = models.UpdateRelease(ctx.Repo.GitRepo, rel, nil); err != nil {
ctx.Handle(500, "UpdateRelease", err)
return
}
}
ctx.JSON(201, rel.APIFormat())
}
@ -111,11 +116,12 @@ func EditRelease(ctx *context.APIContext, form api.EditReleaseOption) {
}
id := ctx.ParamsInt64(":id")
rel, err := models.GetReleaseByID(id)
if err != nil {
if err != nil && !models.IsErrReleaseNotExist(err) {
ctx.Error(500, "GetReleaseByID", err)
return
}
if rel.RepoID != ctx.Repo.Repository.ID {
if err != nil && models.IsErrReleaseNotExist(err) ||
rel.IsTag || rel.RepoID != ctx.Repo.Repository.ID {
ctx.Status(404)
return
}
@ -162,12 +168,13 @@ func DeleteRelease(ctx *context.APIContext) {
return
}
id := ctx.ParamsInt64(":id")
release, err := models.GetReleaseByID(id)
if err != nil {
rel, err := models.GetReleaseByID(id)
if err != nil && !models.IsErrReleaseNotExist(err) {
ctx.Error(500, "GetReleaseByID", err)
return
}
if release.RepoID != ctx.Repo.Repository.ID {
if err != nil && models.IsErrReleaseNotExist(err) ||
rel.IsTag || rel.RepoID != ctx.Repo.Repository.ID {
ctx.Status(404)
return
}

58
routers/repo/release.go
View File

@ -67,6 +67,7 @@ func Releases(ctx *context.Context) {
opts := models.FindReleasesOptions{
IncludeDrafts: ctx.Repo.IsWriter(),
IncludeTags: true,
}
releases, err := models.GetReleasesByRepoID(ctx.Repo.Repository.ID, opts, page, limit)
@ -145,24 +146,15 @@ func NewReleasePost(ctx *context.Context, form auth.NewReleaseForm) {
return
}
var tagCreatedUnix int64
tag, err := ctx.Repo.GitRepo.GetTag(form.TagName)
if err == nil {
commit, err := tag.Commit()
if err == nil {
tagCreatedUnix = commit.Author.When.Unix()
}
var attachmentUUIDs []string
if setting.AttachmentEnabled {
attachmentUUIDs = form.Files
}
commit, err := ctx.Repo.GitRepo.GetBranchCommit(form.Target)
rel, err := models.GetRelease(ctx.Repo.Repository.ID, form.TagName)
if err != nil {
ctx.Handle(500, "GetBranchCommit", err)
return
}
commitsCount, err := commit.CommitsCount()
if err != nil {
ctx.Handle(500, "CommitsCount", err)
if !models.IsErrReleaseNotExist(err) {
ctx.Handle(500, "GetRelease", err)
return
}
@ -172,17 +164,10 @@ func NewReleasePost(ctx *context.Context, form auth.NewReleaseForm) {
Title: form.Title,
TagName: form.TagName,
Target: form.Target,
Sha1: commit.ID.String(),
NumCommits: commitsCount,
Note: form.Content,
IsDraft: len(form.Draft) > 0,
IsPrerelease: form.Prerelease,
CreatedUnix: tagCreatedUnix,
}
var attachmentUUIDs []string
if setting.AttachmentEnabled {
attachmentUUIDs = form.Files
IsTag: false,
}
if err = models.CreateRelease(ctx.Repo.GitRepo, rel, attachmentUUIDs); err != nil {
@ -197,6 +182,26 @@ func NewReleasePost(ctx *context.Context, form auth.NewReleaseForm) {
}
return
}
} else {
if !rel.IsTag {
ctx.Data["Err_TagName"] = true
ctx.RenderWithErr(ctx.Tr("repo.release.tag_name_already_exist"), tplReleaseNew, &form)
return
}
rel.Title = form.Title
rel.Note = form.Content
rel.IsDraft = len(form.Draft) > 0
rel.IsPrerelease = form.Prerelease
rel.PublisherID = ctx.User.ID
rel.IsTag = false
if err = models.UpdateRelease(ctx.Repo.GitRepo, rel, attachmentUUIDs); err != nil {
ctx.Data["Err_TagName"] = true
ctx.Handle(500, "UpdateRelease", err)
return
}
}
log.Trace("Release created: %s/%s:%s", ctx.User.LowerName, ctx.Repo.Repository.Name, form.TagName)
ctx.Redirect(ctx.Repo.RepoLink + "/releases")
@ -246,6 +251,10 @@ func EditReleasePost(ctx *context.Context, form auth.EditReleaseForm) {
}
return
}
if rel.IsTag {
ctx.Handle(404, "GetRelease", err)
return
}
ctx.Data["tag_name"] = rel.TagName
ctx.Data["tag_target"] = rel.Target
ctx.Data["title"] = rel.Title
@ -275,8 +284,7 @@ func EditReleasePost(ctx *context.Context, form auth.EditReleaseForm) {
// DeleteRelease delete a release
func DeleteRelease(ctx *context.Context) {
delTag := ctx.QueryBool("delTag")
if err := models.DeleteReleaseByID(ctx.QueryInt64("id"), ctx.User, delTag); err != nil {
if err := models.DeleteReleaseByID(ctx.QueryInt64("id"), ctx.User, true); err != nil {
ctx.Flash.Error("DeleteReleaseByID: " + err.Error())
} else {
ctx.Flash.Success(ctx.Tr("repo.release.deletion_success"))

25
templates/repo/release/list.tmpl
View File

@ -17,7 +17,9 @@
{{range .Releases}}
<li class="ui grid">
<div class="ui four wide column meta">
{{if .PublisherID}}
{{if .IsTag}}
{{if .Created}}<span class="time">{{TimeSince .Created $.Lang}}</span>{{end}}
{{else}}
{{if .IsDraft}}
<span class="ui yellow label">{{$.i18n.Tr "repo.release.draft"}}</span>
{{else if .IsPrerelease}}
@ -28,13 +30,22 @@
<span class="tag text blue">
<a href="{{$.RepoLink}}/src/{{.TagName}}" rel="nofollow"><i class="tag icon"></i> {{.TagName}}</a>
</span>
{{end}}
<span class="commit">
<a href="{{$.RepoLink}}/src/{{.Sha1}}" rel="nofollow"><i class="code icon"></i> {{ShortSha .Sha1}}</a>
</span>
{{end}}
</div>
<div class="ui twelve wide column detail">
{{if .PublisherID}}
{{if .IsTag}}
<h4>
<a href="{{$.RepoLink}}/src/{{.TagName}}" rel="nofollow"><i class="tag icon"></i> {{.TagName}}</a>
</h4>
<div class="download">
<a href="{{$.RepoLink}}/src/{{.Sha1}}" rel="nofollow"><i class="code icon"></i> {{ShortSha .Sha1}}</a>
<a href="{{$.RepoLink}}/archive/{{.TagName}}.zip" rel="nofollow"><i class="octicon octicon-file-zip"></i> ZIP</a>
<a href="{{$.RepoLink}}/archive/{{.TagName}}.tar.gz"><i class="octicon octicon-file-zip"></i> TAR.GZ</a>
</div>
{{else}}
<h3>
<a href="{{$.RepoLink}}/src/{{.TagName}}">{{.Title}}</a>
{{if $.IsRepositoryWriter}}<small>(<a href="{{$.RepoLink}}/releases/edit/{{.TagName}}" rel="nofollow">{{$.i18n.Tr "repo.release.edit"}}</a>)</small>{{end}}
@ -70,14 +81,6 @@
{{end}}
</ul>
</div>
{{else}}
<h4>
<a href="{{$.RepoLink}}/src/{{.TagName}}" rel="nofollow"><i class="tag icon"></i> {{.TagName}}</a>
</h4>
<div class="download">
<a href="{{$.RepoLink}}/archive/{{.TagName}}.zip" rel="nofollow"><i class="octicon octicon-file-zip"></i> ZIP</a>
<a href="{{$.RepoLink}}/archive/{{.TagName}}.tar.gz"><i class="octicon octicon-file-zip"></i> TAR.GZ</a>
</div>
{{end}}
<span class="dot">&nbsp;</span>
</div>