diff --git a/pkg/cmd/cmd.go b/pkg/cmd/cmd.go index 53a4f790..0b742583 100644 --- a/pkg/cmd/cmd.go +++ b/pkg/cmd/cmd.go @@ -17,15 +17,6 @@ package cmd import ( - "code.vikunja.io/api/pkg/config" - "code.vikunja.io/api/pkg/files" - "code.vikunja.io/api/pkg/log" - "code.vikunja.io/api/pkg/mail" - "code.vikunja.io/api/pkg/migration" - "code.vikunja.io/api/pkg/models" - migrator "code.vikunja.io/api/pkg/modules/migration" - "code.vikunja.io/api/pkg/red" - "code.vikunja.io/api/pkg/user" "fmt" "github.com/spf13/cobra" "os" @@ -54,48 +45,3 @@ func Execute() { os.Exit(1) } } - -// Will only fullInit config, redis, logger but no db connection. -func lightInit() { - // Init the config - config.InitConfig() - - // Init redis - red.InitRedis() - - // Set logger - log.InitLogger() -} - -// Initializes all kinds of things in the right order -func fullInit() { - - lightInit() - - // Run the migrations - migration.Migrate(nil) - - // Set Engine - err := models.SetEngine() - if err != nil { - log.Fatal(err.Error()) - } - err = user.InitDB() - if err != nil { - log.Fatal(err.Error()) - } - err = files.SetEngine() - if err != nil { - log.Fatal(err.Error()) - } - err = migrator.InitDB() - if err != nil { - log.Fatal(err.Error()) - } - - // Initialize the files handler - files.InitFileHandler() - - // Start the mail daemon - mail.StartMailDaemon() -} diff --git a/pkg/cmd/dump.go b/pkg/cmd/dump.go index 34f36b16..235698bd 100644 --- a/pkg/cmd/dump.go +++ b/pkg/cmd/dump.go @@ -17,6 +17,7 @@ package cmd import ( + "code.vikunja.io/api/pkg/initialize" "code.vikunja.io/api/pkg/log" "code.vikunja.io/api/pkg/modules/dump" "github.com/spf13/cobra" @@ -31,7 +32,7 @@ var dumpCmd = &cobra.Command{ Use: "dump", Short: "Dump all vikunja data into a zip file. Includes config, files and db.", PreRun: func(cmd *cobra.Command, args []string) { - fullInit() + initialize.FullInit() }, Run: func(cmd *cobra.Command, args []string) { filename := "vikunja-dump_" + time.Now().Format("2006-01-02_15-03-05") + ".zip" diff --git a/pkg/cmd/migrate.go b/pkg/cmd/migrate.go index a53387c8..016db19f 100644 --- a/pkg/cmd/migrate.go +++ b/pkg/cmd/migrate.go @@ -17,6 +17,7 @@ package cmd import ( + "code.vikunja.io/api/pkg/initialize" "code.vikunja.io/api/pkg/migration" "github.com/spf13/cobra" ) @@ -36,7 +37,7 @@ var migrateCmd = &cobra.Command{ Use: "migrate", Short: "Run all database migrations which didn't already run.", PersistentPreRun: func(cmd *cobra.Command, args []string) { - lightInit() + initialize.LightInit() }, Run: func(cmd *cobra.Command, args []string) { migration.Migrate(nil) diff --git a/pkg/cmd/restore.go b/pkg/cmd/restore.go new file mode 100644 index 00000000..54719d44 --- /dev/null +++ b/pkg/cmd/restore.go @@ -0,0 +1,42 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2020 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package cmd + +import ( + "code.vikunja.io/api/pkg/initialize" + "code.vikunja.io/api/pkg/log" + "code.vikunja.io/api/pkg/modules/dump" + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(restoreCmd) +} + +var restoreCmd = &cobra.Command{ + Use: "restore [filename]", + Short: "Restores all vikunja data from a vikunja dump.", + Args: cobra.ExactArgs(1), + PreRun: func(cmd *cobra.Command, args []string) { + initialize.FullInit() + }, + Run: func(cmd *cobra.Command, args []string) { + if err := dump.Restore(args[0]); err != nil { + log.Critical(err.Error()) + } + }, +} diff --git a/pkg/cmd/testmail.go b/pkg/cmd/testmail.go index 95b9cf51..f02d4b3c 100644 --- a/pkg/cmd/testmail.go +++ b/pkg/cmd/testmail.go @@ -17,6 +17,7 @@ package cmd import ( + "code.vikunja.io/api/pkg/initialize" "code.vikunja.io/api/pkg/log" "code.vikunja.io/api/pkg/mail" "github.com/spf13/cobra" @@ -31,7 +32,7 @@ var testmailCmd = &cobra.Command{ Short: "Send a test mail using the configured smtp connection", Args: cobra.ExactArgs(1), PreRun: func(cmd *cobra.Command, args []string) { - lightInit() + initialize.LightInit() // Start the mail daemon mail.StartMailDaemon() diff --git a/pkg/cmd/web.go b/pkg/cmd/web.go index c2935109..a727cebd 100644 --- a/pkg/cmd/web.go +++ b/pkg/cmd/web.go @@ -18,6 +18,7 @@ package cmd import ( "code.vikunja.io/api/pkg/config" + "code.vikunja.io/api/pkg/initialize" "code.vikunja.io/api/pkg/log" "code.vikunja.io/api/pkg/routes" "code.vikunja.io/api/pkg/swagger" @@ -37,7 +38,7 @@ var webCmd = &cobra.Command{ Use: "web", Short: "Starts the rest api web server", PreRun: func(cmd *cobra.Command, args []string) { - fullInit() + initialize.FullInit() }, Run: func(cmd *cobra.Command, args []string) { diff --git a/pkg/db/db.go b/pkg/db/db.go index 713ace74..457d87fb 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -168,3 +168,20 @@ func initSqliteEngine() (engine *xorm.Engine, err error) { return xorm.NewEngine("sqlite3", path) } + +// WipeEverything wipes all tables and their data. Use with caution... +func WipeEverything() error { + + tables, err := x.DBMetas() + if err != nil { + return err + } + + for _, t := range tables { + if err := x.DropTables(t.Name); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/db/dump.go b/pkg/db/dump.go index f6080164..70bd3f98 100644 --- a/pkg/db/dump.go +++ b/pkg/db/dump.go @@ -40,3 +40,15 @@ func Dump() (data map[string][]byte, err error) { return } + +// Restore restores a table with all its entries +func Restore(table string, contents []map[string]interface{}) (err error) { + + for _, content := range contents { + if _, err := x.Table(table).Insert(content); err != nil { + return err + } + } + + return +} diff --git a/pkg/files/files.go b/pkg/files/files.go index 060f6f5c..94b5505a 100644 --- a/pkg/files/files.go +++ b/pkg/files/files.go @@ -96,7 +96,7 @@ func Create(f io.ReadCloser, realname string, realsize uint64, a web.Auth) (file } // Save the file to storage with its new ID as path - err = afs.WriteReader(file.getFileName(), f) + err = file.Save(f) return } @@ -113,3 +113,8 @@ func (f *File) Delete() (err error) { err = afs.Remove(f.getFileName()) return } + +// Save saves a file to storage +func (f *File) Save(fcontent io.ReadCloser) error { + return afs.WriteReader(f.getFileName(), fcontent) +} diff --git a/pkg/initialize/init.go b/pkg/initialize/init.go new file mode 100644 index 00000000..dc9f89e8 --- /dev/null +++ b/pkg/initialize/init.go @@ -0,0 +1,79 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2020 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package initialize + +import ( + "code.vikunja.io/api/pkg/config" + "code.vikunja.io/api/pkg/files" + "code.vikunja.io/api/pkg/log" + "code.vikunja.io/api/pkg/mail" + "code.vikunja.io/api/pkg/migration" + "code.vikunja.io/api/pkg/models" + migrator "code.vikunja.io/api/pkg/modules/migration" + "code.vikunja.io/api/pkg/red" + "code.vikunja.io/api/pkg/user" +) + +// LightInit will only fullInit config, redis, logger but no db connection. +func LightInit() { + // Init the config + config.InitConfig() + + // Init redis + red.InitRedis() + + // Set logger + log.InitLogger() +} + +// InitEngines intializes all db connections +func InitEngines() { + err := models.SetEngine() + if err != nil { + log.Fatal(err.Error()) + } + err = user.InitDB() + if err != nil { + log.Fatal(err.Error()) + } + err = files.SetEngine() + if err != nil { + log.Fatal(err.Error()) + } + err = migrator.InitDB() + if err != nil { + log.Fatal(err.Error()) + } +} + +// FullInit initializes all kinds of things in the right order +func FullInit() { + + LightInit() + + // Run the migrations + migration.Migrate(nil) + + // Set Engine + InitEngines() + + // Initialize the files handler + files.InitFileHandler() + + // Start the mail daemon + mail.StartMailDaemon() +} diff --git a/pkg/migration/migration.go b/pkg/migration/migration.go index c112e704..c6970bf8 100644 --- a/pkg/migration/migration.go +++ b/pkg/migration/migration.go @@ -104,6 +104,15 @@ func Rollback(migrationID string) { log.Info("Rolled back successfully.") } +// MigrateTo executes all migrations up to a certain point +func MigrateTo(migrationID string, x *xorm.Engine) error { + m := initMigration(x) + if err := m.MigrateTo(migrationID); err != nil { + return err + } + return nil +} + // Deletes a column from a table. All arguments are strings, to let them be standalone and not depending on any struct. func dropTableColum(x *xorm.Engine, tableName, col string) error { diff --git a/pkg/modules/dump/restore.go b/pkg/modules/dump/restore.go new file mode 100644 index 00000000..504458db --- /dev/null +++ b/pkg/modules/dump/restore.go @@ -0,0 +1,217 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2020 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package dump + +import ( + "archive/zip" + "bufio" + "bytes" + "code.vikunja.io/api/pkg/db" + "code.vikunja.io/api/pkg/files" + "code.vikunja.io/api/pkg/initialize" + "code.vikunja.io/api/pkg/log" + "code.vikunja.io/api/pkg/migration" + "encoding/json" + "fmt" + "io" + "os" + "sort" + "src.techknowlogick.com/xormigrate" + "strconv" + "strings" +) + +const maxConfigSize = 5 * 1024 * 1024 // 5 MB, should be largely enough + +// Restore takes a zip file name and restores it +func Restore(filename string) error { + + r, err := zip.OpenReader(filename) + if err != nil { + return fmt.Errorf("could not open zip file: %s", err) + } + + log.Warning("Restoring a dump will wipe your current installation!") + log.Warning("To confirm, please type 'Yes, I understand' and confirm with enter:") + cr := bufio.NewReader(os.Stdin) + text, err := cr.ReadString('\n') + if err != nil { + return fmt.Errorf("could not read confirmation message: %s", err) + } + if text != "Yes, I understand\n" { + return fmt.Errorf("invalid confirmation message") + } + + // Find the configFile, database and files files + var configFile *zip.File + dbfiles := make(map[string]*zip.File) + filesFiles := make(map[string]*zip.File) + for _, file := range r.File { + if strings.HasPrefix(file.Name, "config") { + configFile = file + continue + } + if strings.HasPrefix(file.Name, "database/") { + fname := strings.ReplaceAll(file.Name, "database/", "") + dbfiles[fname[:len(fname)-5]] = file + continue + } + if strings.HasPrefix(file.Name, "files/") { + filesFiles[strings.ReplaceAll(file.Name, "files/", "")] = file + } + } + if configFile == nil { + return fmt.Errorf("dump does not contain a config file") + } + + /////// + // Restore the config file + if configFile.UncompressedSize64 > maxConfigSize { + return fmt.Errorf("config file too large, is %d, max size is %d", configFile.UncompressedSize64, maxConfigSize) + } + + outFile, err := os.OpenFile(configFile.Name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, configFile.Mode()) + if err != nil { + return fmt.Errorf("could not open config file for writing: %s", err) + } + + cfgr, err := configFile.Open() + if err != nil { + return err + } + + // #nosec - We eliminated the potential decompression bomb by erroring out above if the file is larger than a threshold. + _, err = io.Copy(outFile, cfgr) + if err != nil { + return fmt.Errorf("could not create config file: %s", err) + } + + _ = cfgr.Close() + _ = outFile.Close() + + log.Infof("The config file has been restored to '%s'.", configFile.Name) + log.Infof("You can now make changes to it, hit enter when you're done.") + if _, err := bufio.NewReader(os.Stdin).ReadString('\n'); err != nil { + return fmt.Errorf("could not read from stdin: %s", err) + } + log.Info("Restoring...") + + // Init the configFile again since the restored configuration is most likely different from the one before + initialize.LightInit() + initialize.InitEngines() + files.InitFileHandler() + + /////// + // Restore the db + // Start by wiping everything + if err := db.WipeEverything(); err != nil { + return fmt.Errorf("could not wipe database: %s", err) + } + log.Info("Wiped database.") + + // Because we don't explicitly saved the table definitions, we take the last ran db migration from the dump + // and execute everything until that point. + migrations := dbfiles["migration"] + rc, err := migrations.Open() + if err != nil { + return fmt.Errorf("could not open migrations: %s", err) + } + defer rc.Close() + + var buf bytes.Buffer + if _, err := buf.ReadFrom(rc); err != nil { + return fmt.Errorf("could not read migrations: %s", err) + } + + ms := []*xormigrate.Migration{} + if err := json.Unmarshal(buf.Bytes(), &ms); err != nil { + return fmt.Errorf("could not read migrations: %s", err) + } + sort.Slice(ms, func(i, j int) bool { + return ms[i].ID > ms[j].ID + }) + + lastMigration := ms[len(ms)-1] + if err := migration.MigrateTo(lastMigration.ID, nil); err != nil { + return fmt.Errorf("could not create db structure: %s", err) + } + + // Restore all db data + for table, d := range dbfiles { + content, err := unmarshalFileToJSON(d) + if err != nil { + return fmt.Errorf("could not read table %s: %s", table, err) + } + if err := db.Restore(table, content); err != nil { + return fmt.Errorf("could not restore table data for table %s: %s", table, err) + } + log.Infof("Restored table %s", table) + } + log.Infof("Restored %d tables", len(dbfiles)) + + // Run migrations again to migrate a potentially outdated dump + migration.Migrate(nil) + + /////// + // Restore Files + for i, file := range filesFiles { + id, err := strconv.ParseInt(i, 10, 64) + if err != nil { + return fmt.Errorf("could not parse file id %s: %s", i, err) + } + + f := &files.File{ID: id} + + fc, err := file.Open() + if err != nil { + return fmt.Errorf("could not open file %s: %s", i, err) + } + + if err := f.Save(fc); err != nil { + return fmt.Errorf("could not save file: %s", err) + } + + _ = fc.Close() + log.Infof("Restored file %s", i) + } + log.Infof("Restored %d files.", len(filesFiles)) + + /////// + // Done + log.Infof("Done restoring dump.") + + return nil +} + +func unmarshalFileToJSON(file *zip.File) (contents []map[string]interface{}, err error) { + rc, err := file.Open() + if err != nil { + return + } + defer rc.Close() + + var buf bytes.Buffer + if _, err := buf.ReadFrom(rc); err != nil { + return nil, err + } + + contents = []map[string]interface{}{} + if err := json.Unmarshal(buf.Bytes(), &contents); err != nil { + return nil, err + } + return +}