From 971b13c367c6dcaf1cda558e682a73de3965004f Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 15 Nov 2017 17:15:34 +0800 Subject: [PATCH 1/5] Reimplement dump and implment restore command --- cmd/dump.go | 46 +++++++-------- cmd/restore.go | 148 +++++++++++++++++++++++++++++++++++++++++++++++ main.go | 1 + models/models.go | 80 +++++++++++++++++++++++++ 4 files changed, 252 insertions(+), 23 deletions(-) create mode 100644 cmd/restore.go diff --git a/cmd/dump.go b/cmd/dump.go index bbefda63e..1cba91c39 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -1,5 +1,5 @@ // Copyright 2014 The Gogs Authors. All rights reserved. -// Copyright 2016 The Gitea Authors. All rights reserved. +// Copyright 2017 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. @@ -44,10 +44,6 @@ It can be used for backup and capture Gitea server image to send to maintainer`, Value: os.TempDir(), Usage: "Temporary dir path", }, - cli.StringFlag{ - Name: "database, d", - Usage: "Specify the database SQL syntax", - }, }, } @@ -79,23 +75,15 @@ func runDump(ctx *cli.Context) error { os.Setenv("TMPDIR", tmpWorkDir) } - reposDump := path.Join(tmpWorkDir, "gitea-repo.zip") - dbDump := path.Join(tmpWorkDir, "gitea-db.sql") - + dbDump := path.Join(tmpWorkDir, "database") log.Printf("Dumping local repositories...%s", setting.RepoRootPath) zip.Verbose = ctx.Bool("verbose") - if err := zip.PackTo(setting.RepoRootPath, reposDump, true); err != nil { - log.Fatalf("Failed to dump local repositories: %v", err) + if err := os.MkdirAll(dbDump, os.ModePerm); err != nil { + log.Fatalf("Failed to create database dir: %v", err) } - targetDBType := ctx.String("database") - if len(targetDBType) > 0 && targetDBType != models.DbCfg.Type { - log.Printf("Dumping database %s => %s...", models.DbCfg.Type, targetDBType) - } else { - log.Printf("Dumping database...") - } - - if err := models.DumpDatabase(dbDump, targetDBType); err != nil { + log.Printf("Dumping database ...") + if err := models.DumpDatabaseFixtures(dbDump); err != nil { log.Fatalf("Failed to dump database: %v", err) } @@ -106,11 +94,12 @@ func runDump(ctx *cli.Context) error { log.Fatalf("Failed to create %s: %v", fileName, err) } - if err := z.AddFile("gitea-repo.zip", reposDump); err != nil { + if err := z.AddDir("repositories", setting.RepoRootPath); err != nil { log.Fatalf("Failed to include gitea-repo.zip: %v", err) } - if err := z.AddFile("gitea-db.sql", dbDump); err != nil { - log.Fatalf("Failed to include gitea-db.sql: %v", err) + + if err := z.AddDir("database", dbDump); err != nil { + log.Fatalf("Failed to include database: %v", err) } customDir, err := os.Stat(setting.CustomPath) if err == nil && customDir.IsDir() { @@ -133,8 +122,19 @@ func runDump(ctx *cli.Context) error { } } - if err := z.AddDir("log", setting.LogRootPath); err != nil { - log.Fatalf("Failed to include log: %v", err) + verPath := filepath.Join(tmpWorkDir, "VERSION") + verf, err := os.Create(verPath) + if err != nil { + log.Fatalf("Failed to create version file: %v", err) + } + _, err = verf.WriteString(setting.AppVer) + verf.Close() + if err != nil { + log.Fatalf("Failed to write version to file: %v", err) + } + + if err = z.AddFile("VERSION", verPath); err != nil { + log.Fatalf("Failed to add version file: %v", err) } if err = z.Close(); err != nil { diff --git a/cmd/restore.go b/cmd/restore.go new file mode 100644 index 000000000..00341c0c3 --- /dev/null +++ b/cmd/restore.go @@ -0,0 +1,148 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "errors" + "io/ioutil" + "log" + "os" + "path/filepath" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/setting" + + "github.com/Unknwon/cae/zip" + "github.com/Unknwon/com" + "github.com/urfave/cli" +) + +// CmdRestore represents the available restore sub-command. +var CmdRestore = cli.Command{ + Name: "restore", + Usage: "Restore Gitea files and database", + Description: `Restore will restore all data from zip file which dumped from gitea. It will use +the custom config in this dump zip file, this operation will remove all the dest database and repositories.`, + Action: runRestore, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "config, c", + Value: "custom/conf/app.ini", + Usage: "Custom configuration file path, if empty will use dumped config file", + }, + cli.BoolFlag{ + Name: "verbose, v", + Usage: "Show process details", + }, + cli.StringFlag{ + Name: "tempdir, t", + Value: os.TempDir(), + Usage: "Temporary dir path", + }, + }, +} + +func runRestore(ctx *cli.Context) error { + if len(os.Args) < 3 { + return errors.New("need zip file path") + } + + tmpDir := ctx.String("tempdir") + if _, err := os.Stat(tmpDir); os.IsNotExist(err) { + log.Fatalf("Path does not exist: %s", tmpDir) + } + tmpWorkDir, err := ioutil.TempDir(tmpDir, "gitea-dump-") + if err != nil { + log.Fatalf("Failed to create tmp work directory: %v", err) + } + log.Printf("Creating tmp work dir: %s", tmpWorkDir) + + // work-around #1103 + if os.Getenv("TMPDIR") == "" { + os.Setenv("TMPDIR", tmpWorkDir) + } + + srcPath := os.Args[2] + + zip.Verbose = ctx.Bool("verbose") + log.Printf("Extracting %s to tmp work dir", srcPath) + err = zip.ExtractTo(srcPath, tmpWorkDir) + if err != nil { + log.Fatalf("Failed to extract %s to tmp work directory: %v", srcPath, err) + } + + verData, err := ioutil.ReadFile(filepath.Join(tmpWorkDir, "VERSION")) + if err != nil { + log.Fatalf("Failed to extract %s to tmp work directory: %v", srcPath, err) + } + + if setting.AppVer != string(verData) { + log.Fatalf("Expected gitea version to restore is %s, but get %s", string(verData), setting.AppVer) + } + + if ctx.IsSet("config") { + setting.CustomConf = ctx.String("config") + } else { + setting.CustomConf = filepath.Join(tmpWorkDir, "custom", "conf", "app.ini") + } + if !com.IsExist(setting.CustomConf) { + log.Fatalf("Failed to load ini config file from %s", setting.CustomConf) + } + + setting.NewContext() + //setting.CustomPath = filepath.Join(tmpWorkDir, "custom") + setting.NewXORMLogService(false) + models.LoadConfigs() + + err = models.SetEngine() + if err != nil { + log.Fatalf("Failed to SetEngine: %v", err) + } + + log.Printf("Restoring repo dir %s ...", setting.RepoRootPath) + repoPath := filepath.Join(tmpWorkDir, "repositories") + err = os.RemoveAll(setting.RepoRootPath) + if err != nil { + log.Fatalf("Failed to Remove repo root path %s: %v", setting.RepoRootPath, err) + } + + err = os.Rename(repoPath, setting.RepoRootPath) + if err != nil { + log.Fatalf("Failed to move %s to %s: %v", repoPath, setting.RepoRootPath, err) + } + + log.Printf("Restoring custom dir %s ...", setting.CustomPath) + customPath := filepath.Join(tmpWorkDir, "custom") + err = os.RemoveAll(setting.CustomPath) + if err != nil { + log.Fatalf("Failed to Remove repo root path %s: %v", setting.CustomPath, err) + } + + err = os.Rename(customPath, setting.CustomPath) + if err != nil { + log.Fatalf("Failed to move %s to %s: %v", customPath, setting.CustomPath, err) + } + + log.Printf("Restoring data dir %s ...", setting.AppDataPath) + dataPath := filepath.Join(tmpWorkDir, "data") + err = os.RemoveAll(setting.AppDataPath) + if err != nil { + log.Fatalf("Failed to Remove data root path %s: %v", setting.AppDataPath, err) + } + + err = os.Rename(dataPath, setting.AppDataPath) + if err != nil { + log.Fatalf("Failed to move %s to %s: %v", dataPath, setting.AppDataPath, err) + } + + log.Printf("Restoring database from ...") + dbPath := filepath.Join(tmpWorkDir, "database") + err = models.RestoreDatabaseFixtures(dbPath) + if err != nil { + log.Fatalf("Failed to restore database dir %s: %v", dbPath, err) + } + + return nil +} diff --git a/main.go b/main.go index 179132f58..f1a5da5d3 100644 --- a/main.go +++ b/main.go @@ -43,6 +43,7 @@ arguments - which can alternatively be run by running the subcommand web.` cmd.CmdServ, cmd.CmdHook, cmd.CmdDump, + cmd.CmdRestore, cmd.CmdCert, cmd.CmdAdmin, cmd.CmdGenerate, diff --git a/models/models.go b/models/models.go index 549d2eadc..bb1b2d5e3 100644 --- a/models/models.go +++ b/models/models.go @@ -8,10 +8,12 @@ import ( "database/sql" "errors" "fmt" + "io/ioutil" "net/url" "os" "path" "path/filepath" + "reflect" "strings" "code.gitea.io/gitea/modules/log" @@ -21,6 +23,7 @@ import ( _ "github.com/go-sql-driver/mysql" "github.com/go-xorm/core" "github.com/go-xorm/xorm" + "gopkg.in/yaml.v2" // Needed for the Postgresql driver _ "github.com/lib/pq" @@ -352,3 +355,80 @@ func DumpDatabase(filePath string, dbType string) error { } return x.DumpTablesToFile(tbs, filePath) } + +// DumpDatabaseFixtures dumps all data from database to fixtures files on dirPath +func DumpDatabaseFixtures(dirPath string) error { + for _, t := range tables { + if err := dumpTableFixtures(t, dirPath); err != nil { + return err + } + } + return nil +} + +func dumpTableFixtures(bean interface{}, dirPath string) error { + table := x.TableInfo(bean) + f, err := os.Create(filepath.Join(dirPath, table.Name+".yml")) + if err != nil { + return err + } + defer f.Close() + var bufferSize = 100 + var objs = make([]interface{}, 0, bufferSize) + err = x.BufferSize(bufferSize).Iterate(bean, func(idx int, obj interface{}) error { + objs = append(objs, obj) + if len(objs) == bufferSize { + // BLOCK: need yaml support gonic name mapper + data, err := yaml.Marshal(objs) + if err != nil { + return err + } + _, err = f.Write(data) + if err != nil { + return err + } + objs = make([]interface{}, 0, bufferSize) + } + return err + }) + if err != nil { + return err + } + if len(objs) > 0 { + data, err := yaml.Marshal(objs) + if err != nil { + return err + } + _, err = f.Write(data) + } + return err +} + +// RestoreDatabaseFixtures restores all data from dir to database +func RestoreDatabaseFixtures(dirPath string) error { + for _, t := range tables { + if err := restoreTableFixtures(t, dirPath); err != nil { + return err + } + } + return nil +} + +func restoreTableFixtures(bean interface{}, dirPath string) error { + table := x.TableInfo(bean) + data, err := ioutil.ReadFile(filepath.Join(dirPath, table.Name+".yml")) + if err != nil { + return err + } + + var bufferSize = 100 + v := reflect.MakeSlice(table.Type, 0, bufferSize) + // BLOCK: need yaml support gonic name mapper + err = yaml.Unmarshal(data, v.Interface()) + if err != nil { + return err + } + + _, err = x.Insert(v.Interface()) + return err +} From 7810e439d4daae0acd88c456495d688ccb61695b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 8 May 2018 07:07:16 +0800 Subject: [PATCH 2/5] fix name convert --- models/models.go | 84 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 56 insertions(+), 28 deletions(-) diff --git a/models/models.go b/models/models.go index bb1b2d5e3..5ac109ea1 100644 --- a/models/models.go +++ b/models/models.go @@ -13,7 +13,7 @@ import ( "os" "path" "path/filepath" - "reflect" + "sort" "strings" "code.gitea.io/gitea/modules/log" @@ -373,35 +373,33 @@ func dumpTableFixtures(bean interface{}, dirPath string) error { return err } defer f.Close() - var bufferSize = 100 - var objs = make([]interface{}, 0, bufferSize) - err = x.BufferSize(bufferSize).Iterate(bean, func(idx int, obj interface{}) error { - objs = append(objs, obj) - if len(objs) == bufferSize { - // BLOCK: need yaml support gonic name mapper - data, err := yaml.Marshal(objs) - if err != nil { - return err - } - _, err = f.Write(data) - if err != nil { - return err - } - objs = make([]interface{}, 0, bufferSize) + + const bufferSize = 100 + var start = 0 + for { + objs, err := x.Table(table.Name).Limit(bufferSize, start).QueryInterface() + if err != nil { + return err } - return err - }) - if err != nil { - return err - } - if len(objs) > 0 { + if len(objs) == 0 { + break + } + data, err := yaml.Marshal(objs) if err != nil { return err } _, err = f.Write(data) + if err != nil { + return err + } + if len(objs) < bufferSize { + break + } + start += len(objs) } - return err + + return nil } // RestoreDatabaseFixtures restores all data from dir to database @@ -421,14 +419,44 @@ func restoreTableFixtures(bean interface{}, dirPath string) error { return err } - var bufferSize = 100 - v := reflect.MakeSlice(table.Type, 0, bufferSize) - // BLOCK: need yaml support gonic name mapper - err = yaml.Unmarshal(data, v.Interface()) + const bufferSize = 100 + var records = make([]map[string]interface{}, 0, bufferSize*10) + err = yaml.Unmarshal(data, records) if err != nil { return err } - _, err = x.Insert(v.Interface()) + if len(records) == 0 { + return nil + } + + var columns = make([]string, 0, len(records[0])) + for k, _ := range records[0] { + columns = append(columns, k) + } + sort.Strings(columns) + + qm := strings.Repeat("?,", len(columns)) + qm = "(" + qm[:len(qm)-1] + ")" + + var sql = "INSERT INTO " + table.Name + "(" + strings.Join(columns, ",") + ") VALUES " + var args = make([]interface{}, 0, bufferSize) + var insertSQLs = make([]string, 0, bufferSize) + for i, vals := range records { + insertSQLs = append(insertSQLs, qm) + for _, colName := range columns { + args = append(args, vals[colName]) + } + + if i+1%100 == 0 || i == len(records)-1 { + _, err = x.Exec(sql+strings.Join(insertSQLs, ","), args...) + if err != nil { + return err + } + insertSQLs = make([]string, 0, bufferSize) + args = make([]interface{}, 0, bufferSize) + } + } + return err } From 9a9a0bc1b8ce00bd79a4a20e64f18e360758bc4a Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 8 May 2018 16:29:57 +0800 Subject: [PATCH 3/5] fix lint --- cmd/dump.go | 2 +- models/models.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/dump.go b/cmd/dump.go index 1cba91c39..ed4d3ecee 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -1,5 +1,5 @@ // Copyright 2014 The Gogs Authors. All rights reserved. -// Copyright 2017 The Gitea Authors. All rights reserved. +// Copyright 2016 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. diff --git a/models/models.go b/models/models.go index 5ac109ea1..69bfe7664 100644 --- a/models/models.go +++ b/models/models.go @@ -431,7 +431,7 @@ func restoreTableFixtures(bean interface{}, dirPath string) error { } var columns = make([]string, 0, len(records[0])) - for k, _ := range records[0] { + for k := range records[0] { columns = append(columns, k) } sort.Strings(columns) From 22edc1633695b2c55eeadd6a67a738a284a54de5 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 8 May 2018 17:37:52 +0800 Subject: [PATCH 4/5] fix typo --- cmd/restore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/restore.go b/cmd/restore.go index 00341c0c3..cdba8b8bb 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -53,7 +53,7 @@ func runRestore(ctx *cli.Context) error { if _, err := os.Stat(tmpDir); os.IsNotExist(err) { log.Fatalf("Path does not exist: %s", tmpDir) } - tmpWorkDir, err := ioutil.TempDir(tmpDir, "gitea-dump-") + tmpWorkDir, err := ioutil.TempDir(tmpDir, "gitea-restore-") if err != nil { log.Fatalf("Failed to create tmp work directory: %v", err) } From 9434bc7a22c88e50a7f488662879353ea6bb2433 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 8 May 2018 18:08:26 +0800 Subject: [PATCH 5/5] fix restore --- cmd/restore.go | 15 ++++++++++----- models/models.go | 18 ++++++++++++++++-- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/cmd/restore.go b/cmd/restore.go index cdba8b8bb..f6d26047a 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -67,7 +67,7 @@ func runRestore(ctx *cli.Context) error { srcPath := os.Args[2] zip.Verbose = ctx.Bool("verbose") - log.Printf("Extracting %s to tmp work dir", srcPath) + log.Printf("Extracting %s to %s", srcPath, tmpWorkDir) err = zip.ExtractTo(srcPath, tmpWorkDir) if err != nil { log.Fatalf("Failed to extract %s to tmp work directory: %v", srcPath, err) @@ -101,7 +101,12 @@ func runRestore(ctx *cli.Context) error { log.Fatalf("Failed to SetEngine: %v", err) } - log.Printf("Restoring repo dir %s ...", setting.RepoRootPath) + err = models.SyncDBStructs() + if err != nil { + log.Fatalf("Failed to SyncDBStructs: %v", err) + } + + log.Printf("Restoring repo dir to %s ...", setting.RepoRootPath) repoPath := filepath.Join(tmpWorkDir, "repositories") err = os.RemoveAll(setting.RepoRootPath) if err != nil { @@ -113,7 +118,7 @@ func runRestore(ctx *cli.Context) error { log.Fatalf("Failed to move %s to %s: %v", repoPath, setting.RepoRootPath, err) } - log.Printf("Restoring custom dir %s ...", setting.CustomPath) + log.Printf("Restoring custom dir to %s ...", setting.CustomPath) customPath := filepath.Join(tmpWorkDir, "custom") err = os.RemoveAll(setting.CustomPath) if err != nil { @@ -125,7 +130,7 @@ func runRestore(ctx *cli.Context) error { log.Fatalf("Failed to move %s to %s: %v", customPath, setting.CustomPath, err) } - log.Printf("Restoring data dir %s ...", setting.AppDataPath) + log.Printf("Restoring data dir to %s ...", setting.AppDataPath) dataPath := filepath.Join(tmpWorkDir, "data") err = os.RemoveAll(setting.AppDataPath) if err != nil { @@ -137,8 +142,8 @@ func runRestore(ctx *cli.Context) error { log.Fatalf("Failed to move %s to %s: %v", dataPath, setting.AppDataPath, err) } - log.Printf("Restoring database from ...") dbPath := filepath.Join(tmpWorkDir, "database") + log.Printf("Restoring database from %s ...", dbPath) err = models.RestoreDatabaseFixtures(dbPath) if err != nil { log.Fatalf("Failed to restore database dir %s: %v", dbPath, err) diff --git a/models/models.go b/models/models.go index 69bfe7664..3b89fd2a9 100644 --- a/models/models.go +++ b/models/models.go @@ -298,6 +298,15 @@ func NewEngine(migrateFunc func(*xorm.Engine) error) (err error) { return nil } +// SyncDBStructs will sync database structs +func SyncDBStructs() error { + if err := x.StoreEngine("InnoDB").Sync2(tables...); err != nil { + return fmt.Errorf("sync database struct error: %v", err) + } + + return nil +} + // Statistic contains the database statistics type Statistic struct { Counter struct { @@ -421,7 +430,7 @@ func restoreTableFixtures(bean interface{}, dirPath string) error { const bufferSize = 100 var records = make([]map[string]interface{}, 0, bufferSize*10) - err = yaml.Unmarshal(data, records) + err = yaml.Unmarshal(data, &records) if err != nil { return err } @@ -439,7 +448,12 @@ func restoreTableFixtures(bean interface{}, dirPath string) error { qm := strings.Repeat("?,", len(columns)) qm = "(" + qm[:len(qm)-1] + ")" - var sql = "INSERT INTO " + table.Name + "(" + strings.Join(columns, ",") + ") VALUES " + _, err = x.Exec("DELETE FROM `" + table.Name + "`") + if err != nil { + return err + } + + var sql = "INSERT INTO `" + table.Name + "` (`" + strings.Join(columns, "`,`") + "`) VALUES " var args = make([]interface{}, 0, bufferSize) var insertSQLs = make([]string, 0, bufferSize) for i, vals := range records {