peter/go-gitea
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge 9434bc7a22 into d0fef4395f

Browse Source
This commit is contained in:
Lunny Xiao 2018-07-24 18:33:29 +00:00 committed by GitHub
commit 307961dad7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 298 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
cmd/dump.go
View File

@ -44,10 +44,6 @@ It can be used for backup and capture Gitea server image to send to maintainer`,
Value: os.TempDir(),
Usage: "Temporary dir path",
},
cli.StringFlag{
Name: "database, d",
Usage: "Specify the database SQL syntax",
},
},
}
@ -79,23 +75,15 @@ func runDump(ctx *cli.Context) error {
os.Setenv("TMPDIR", tmpWorkDir)
}
reposDump := path.Join(tmpWorkDir, "gitea-repo.zip")
dbDump := path.Join(tmpWorkDir, "gitea-db.sql")
dbDump := path.Join(tmpWorkDir, "database")
log.Printf("Dumping local repositories...%s", setting.RepoRootPath)
zip.Verbose = ctx.Bool("verbose")
if err := zip.PackTo(setting.RepoRootPath, reposDump, true); err != nil {
log.Fatalf("Failed to dump local repositories: %v", err)
if err := os.MkdirAll(dbDump, os.ModePerm); err != nil {
log.Fatalf("Failed to create database dir: %v", err)
}
targetDBType := ctx.String("database")
if len(targetDBType) > 0 && targetDBType != models.DbCfg.Type {
log.Printf("Dumping database %s => %s...", models.DbCfg.Type, targetDBType)
} else {
log.Printf("Dumping database...")
}
if err := models.DumpDatabase(dbDump, targetDBType); err != nil {
log.Printf("Dumping database ...")
if err := models.DumpDatabaseFixtures(dbDump); err != nil {
log.Fatalf("Failed to dump database: %v", err)
}
@ -106,11 +94,12 @@ func runDump(ctx *cli.Context) error {
log.Fatalf("Failed to create %s: %v", fileName, err)
}
if err := z.AddFile("gitea-repo.zip", reposDump); err != nil {
if err := z.AddDir("repositories", setting.RepoRootPath); err != nil {
log.Fatalf("Failed to include gitea-repo.zip: %v", err)
}
if err := z.AddFile("gitea-db.sql", dbDump); err != nil {
log.Fatalf("Failed to include gitea-db.sql: %v", err)
if err := z.AddDir("database", dbDump); err != nil {
log.Fatalf("Failed to include database: %v", err)
}
customDir, err := os.Stat(setting.CustomPath)
if err == nil && customDir.IsDir() {
@ -133,8 +122,19 @@ func runDump(ctx *cli.Context) error {
}
}
if err := z.AddDir("log", setting.LogRootPath); err != nil {
log.Fatalf("Failed to include log: %v", err)
verPath := filepath.Join(tmpWorkDir, "VERSION")
verf, err := os.Create(verPath)
if err != nil {
log.Fatalf("Failed to create version file: %v", err)
}
_, err = verf.WriteString(setting.AppVer)
verf.Close()
if err != nil {
log.Fatalf("Failed to write version to file: %v", err)
}
if err = z.AddFile("VERSION", verPath); err != nil {
log.Fatalf("Failed to add version file: %v", err)
}
if err = z.Close(); err != nil {

153
cmd/restore.go Normal file
View File

@ -0,0 +1,153 @@
// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package cmd
import (
"errors"
"io/ioutil"
"log"
"os"
"path/filepath"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/setting"
"github.com/Unknwon/cae/zip"
"github.com/Unknwon/com"
"github.com/urfave/cli"
)
// CmdRestore represents the available restore sub-command.
var CmdRestore = cli.Command{
Name: "restore",
Usage: "Restore Gitea files and database",
Description: `Restore will restore all data from zip file which dumped from gitea. It will use
the custom config in this dump zip file, this operation will remove all the dest database and repositories.`,
Action: runRestore,
Flags: []cli.Flag{
cli.StringFlag{
Name: "config, c",
Value: "custom/conf/app.ini",
Usage: "Custom configuration file path, if empty will use dumped config file",
},
cli.BoolFlag{
Name: "verbose, v",
Usage: "Show process details",
},
cli.StringFlag{
Name: "tempdir, t",
Value: os.TempDir(),
Usage: "Temporary dir path",
},
},
}
func runRestore(ctx *cli.Context) error {
if len(os.Args) < 3 {
return errors.New("need zip file path")
}
tmpDir := ctx.String("tempdir")
if _, err := os.Stat(tmpDir); os.IsNotExist(err) {
log.Fatalf("Path does not exist: %s", tmpDir)
}
tmpWorkDir, err := ioutil.TempDir(tmpDir, "gitea-restore-")
if err != nil {
log.Fatalf("Failed to create tmp work directory: %v", err)
}
log.Printf("Creating tmp work dir: %s", tmpWorkDir)
// work-around #1103
if os.Getenv("TMPDIR") == "" {
os.Setenv("TMPDIR", tmpWorkDir)
}
srcPath := os.Args[2]
zip.Verbose = ctx.Bool("verbose")
log.Printf("Extracting %s to %s", srcPath, tmpWorkDir)
err = zip.ExtractTo(srcPath, tmpWorkDir)
if err != nil {
log.Fatalf("Failed to extract %s to tmp work directory: %v", srcPath, err)
}
verData, err := ioutil.ReadFile(filepath.Join(tmpWorkDir, "VERSION"))
if err != nil {
log.Fatalf("Failed to extract %s to tmp work directory: %v", srcPath, err)
}
if setting.AppVer != string(verData) {
log.Fatalf("Expected gitea version to restore is %s, but get %s", string(verData), setting.AppVer)
}
if ctx.IsSet("config") {
setting.CustomConf = ctx.String("config")
} else {
setting.CustomConf = filepath.Join(tmpWorkDir, "custom", "conf", "app.ini")
}
if !com.IsExist(setting.CustomConf) {
log.Fatalf("Failed to load ini config file from %s", setting.CustomConf)
}
setting.NewContext()
//setting.CustomPath = filepath.Join(tmpWorkDir, "custom")
setting.NewXORMLogService(false)
models.LoadConfigs()
err = models.SetEngine()
if err != nil {
log.Fatalf("Failed to SetEngine: %v", err)
}
err = models.SyncDBStructs()
if err != nil {
log.Fatalf("Failed to SyncDBStructs: %v", err)
}
log.Printf("Restoring repo dir to %s ...", setting.RepoRootPath)
repoPath := filepath.Join(tmpWorkDir, "repositories")
err = os.RemoveAll(setting.RepoRootPath)
if err != nil {
log.Fatalf("Failed to Remove repo root path %s: %v", setting.RepoRootPath, err)
}
err = os.Rename(repoPath, setting.RepoRootPath)
if err != nil {
log.Fatalf("Failed to move %s to %s: %v", repoPath, setting.RepoRootPath, err)
}
log.Printf("Restoring custom dir to %s ...", setting.CustomPath)
customPath := filepath.Join(tmpWorkDir, "custom")
err = os.RemoveAll(setting.CustomPath)
if err != nil {
log.Fatalf("Failed to Remove repo root path %s: %v", setting.CustomPath, err)
}
err = os.Rename(customPath, setting.CustomPath)
if err != nil {
log.Fatalf("Failed to move %s to %s: %v", customPath, setting.CustomPath, err)
}
log.Printf("Restoring data dir to %s ...", setting.AppDataPath)
dataPath := filepath.Join(tmpWorkDir, "data")
err = os.RemoveAll(setting.AppDataPath)
if err != nil {
log.Fatalf("Failed to Remove data root path %s: %v", setting.AppDataPath, err)
}
err = os.Rename(dataPath, setting.AppDataPath)
if err != nil {
log.Fatalf("Failed to move %s to %s: %v", dataPath, setting.AppDataPath, err)
}
dbPath := filepath.Join(tmpWorkDir, "database")
log.Printf("Restoring database from %s ...", dbPath)
err = models.RestoreDatabaseFixtures(dbPath)
if err != nil {
log.Fatalf("Failed to restore database dir %s: %v", dbPath, err)
}
return nil
}

1
main.go
View File

@ -44,6 +44,7 @@ arguments - which can alternatively be run by running the subcommand web.`
cmd.CmdServ,
cmd.CmdHook,
cmd.CmdDump,
cmd.CmdRestore,
cmd.CmdCert,
cmd.CmdAdmin,
cmd.CmdGenerate,

122
models/models.go
View File

@ -9,10 +9,12 @@ import (
"database/sql"
"errors"
"fmt"
"io/ioutil"
"net/url"
"os"
"path"
"path/filepath"
"sort"
"strings"
"code.gitea.io/gitea/modules/log"
@ -22,6 +24,7 @@ import (
_ "github.com/go-sql-driver/mysql"
"github.com/go-xorm/core"
"github.com/go-xorm/xorm"
"gopkg.in/yaml.v2"
// Needed for the Postgresql driver
_ "github.com/lib/pq"
@ -305,6 +308,15 @@ func NewEngine(migrateFunc func(*xorm.Engine) error) (err error) {
return nil
}
// SyncDBStructs will sync database structs
func SyncDBStructs() error {
if err := x.StoreEngine("InnoDB").Sync2(tables...); err != nil {
return fmt.Errorf("sync database struct error: %v", err)
}
return nil
}
// Statistic contains the database statistics
type Statistic struct {
Counter struct {
@ -362,3 +374,113 @@ func DumpDatabase(filePath string, dbType string) error {
}
return x.DumpTablesToFile(tbs, filePath)
}
// DumpDatabaseFixtures dumps all data from database to fixtures files on dirPath
func DumpDatabaseFixtures(dirPath string) error {
for _, t := range tables {
if err := dumpTableFixtures(t, dirPath); err != nil {
return err
}
}
return nil
}
func dumpTableFixtures(bean interface{}, dirPath string) error {
table := x.TableInfo(bean)
f, err := os.Create(filepath.Join(dirPath, table.Name+".yml"))
if err != nil {
return err
}
defer f.Close()
const bufferSize = 100
var start = 0
for {
objs, err := x.Table(table.Name).Limit(bufferSize, start).QueryInterface()
if err != nil {
return err
}
if len(objs) == 0 {
break
}
data, err := yaml.Marshal(objs)
if err != nil {
return err
}
_, err = f.Write(data)
if err != nil {
return err
}
if len(objs) < bufferSize {
break
}
start += len(objs)
}
return nil
}
// RestoreDatabaseFixtures restores all data from dir to database
func RestoreDatabaseFixtures(dirPath string) error {
for _, t := range tables {
if err := restoreTableFixtures(t, dirPath); err != nil {
return err
}
}
return nil
}
func restoreTableFixtures(bean interface{}, dirPath string) error {
table := x.TableInfo(bean)
data, err := ioutil.ReadFile(filepath.Join(dirPath, table.Name+".yml"))
if err != nil {
return err
}
const bufferSize = 100
var records = make([]map[string]interface{}, 0, bufferSize*10)
err = yaml.Unmarshal(data, &records)
if err != nil {
return err
}
if len(records) == 0 {
return nil
}
var columns = make([]string, 0, len(records[0]))
for k := range records[0] {
columns = append(columns, k)
}
sort.Strings(columns)
qm := strings.Repeat("?,", len(columns))
qm = "(" + qm[:len(qm)-1] + ")"
_, err = x.Exec("DELETE FROM `" + table.Name + "`")
if err != nil {
return err
}
var sql = "INSERT INTO `" + table.Name + "` (`" + strings.Join(columns, "`,`") + "`) VALUES "
var args = make([]interface{}, 0, bufferSize)
var insertSQLs = make([]string, 0, bufferSize)
for i, vals := range records {
insertSQLs = append(insertSQLs, qm)
for _, colName := range columns {
args = append(args, vals[colName])
}
if i+1%100 == 0 || i == len(records)-1 {
_, err = x.Exec(sql+strings.Join(insertSQLs, ","), args...)
if err != nil {
return err
}
insertSQLs = make([]string, 0, bufferSize)
args = make([]interface{}, 0, bufferSize)
}
}
return err
}