diff --git a/docs/config.yml b/docs/config.yml index c92270fd..9e1ae794 100644 --- a/docs/config.yml +++ b/docs/config.yml @@ -31,10 +31,10 @@ menu: url: https://vikunja.io/en/ weight: 10 - name: Features - url: https://vikunja.io/en/features + url: https://vikunja.io/features weight: 20 - name: Download - url: https://vikunja.io/en/download + url: https://vikunja.io/download weight: 30 - name: Docs url: https://vikunja.io/docs diff --git a/docs/content/doc/development/migration.md b/docs/content/doc/development/migration.md index b05f5c0e..610ab803 100644 --- a/docs/content/doc/development/migration.md +++ b/docs/content/doc/development/migration.md @@ -14,7 +14,17 @@ It is possible to migrate data from other to-do services to Vikunja. To make this easier, we have put together a few helpers which are documented on this page. In general, each migrator implements a migrator interface which is then called from a client. -The interface makes it possible to use helper methods which handle http an focus only on the implementation of the migrator itself. +The interface makes it possible to use helper methods which handle http and focus only on the implementation of the migrator itself. + +There are two ways of migrating data from another service: +1. Through the auth-based flow where the user gives you access to their data at the third-party service through an + oauth flow. You can then call the service's api on behalf of your user to get all the data. + The Todoist, Trello and Microsoft To-Do Migrators use this pattern. +2. A file migration where the user uploads a file obtained from some third-party service. In your migrator, you need + to parse the file and create the lists, tasks etc. + The Vikunja File Import uses this pattern. + +To differentiate the two, there are two different interfaces you must implement. {{< table_of_contents >}} @@ -23,13 +33,16 @@ The interface makes it possible to use helper methods which handle http an focus All migrator implementations live in their own package in `pkg/modules/migration/`. When creating a new migrator, you should place all related code inside that module. -## Migrator interface +## Migrator Interface The migrator interface is defined as follows: ```go // Migrator is the basic migrator interface which is shared among all migrators type Migrator interface { + // Name holds the name of the migration. + // This is used to show the name to users and to keep track of users who already migrated. + Name() string // Migrate is the interface used to migrate a user's tasks from another platform to vikunja. // The user object is the user who's tasks will be migrated. Migrate(user *models.User) error @@ -37,9 +50,20 @@ type Migrator interface { // The use case for this are Oauth flows, where the server token should remain hidden and not // known to the frontend. AuthURL() string +} +``` + +## File Migrator Interface + +```go +// FileMigrator handles importing Vikunja data from a file. The implementation of it determines the format. +type FileMigrator interface { // Name holds the name of the migration. // This is used to show the name to users and to keep track of users who already migrated. Name() string + // Migrate is the interface used to migrate a user's tasks, list and other things from a file to vikunja. + // The user object is the user who's tasks will be migrated. + Migrate(user *user.User, file io.ReaderAt, size int64) error } ``` @@ -54,15 +78,26 @@ authUrl, Status and Migrate methods. ```go // This is an example for the Wunderlist migrator if config.MigrationWunderlistEnable.GetBool() { - wunderlistMigrationHandler := &migrationHandler.MigrationWeb{ + wunderlistMigrationHandler := &migrationHandler.MigrationWeb{ MigrationStruct: func() migration.Migrator { return &wunderlist.Migration{} }, } - wunderlistMigrationHandler.RegisterRoutes(m) + wunderlistMigrationHandler.RegisterRoutes(m) } ``` +And for the file migrator: + +```go +vikunjaFileMigrationHandler := &migrationHandler.FileMigratorWeb{ + MigrationStruct: func() migration.FileMigrator { + return &vikunja_file.FileMigrator{} + }, +} +vikunjaFileMigrationHandler.RegisterRoutes(m) +``` + You should also document the routes with [swagger annotations]({{< ref "swagger-docs.md" >}}). ## Insertion helper method @@ -70,7 +105,8 @@ You should also document the routes with [swagger annotations]({{< ref "swagger- There is a method available in the `migration` package which takes a fully nested Vikunja structure and creates it with all relations. This means you start by adding a namespace, then add lists inside of that namespace, then tasks in the lists and so on. -The root structure must be present as `[]*models.NamespaceWithLists`. +The root structure must be present as `[]*models.NamespaceWithListsAndTasks`. It allows to represent all of Vikunja's +hierachie as a single data structure. Then call the method like so: @@ -85,14 +121,16 @@ err = migration.InsertFromStructure(fullVikunjaHierachie, user) ## Configuration -You should add at least an option to enable or disable the migration. +If your migrator is an oauth-based one, you should add at least an option to enable or disable it. Chances are, you'll need some more options for things like client ID and secret (if the other service uses oAuth as an authentication flow). The easiest way to implement an on/off switch is to check whether your migration service is enabled or not when -registering the routes, and then simply don't registering the routes in the case it is disabled. +registering the routes, and then simply don't registering the routes in case it is disabled. + +File based migrators can always be enabled. ### Making the migrator public in `/info` You should make your migrator available in the `/info` endpoint so that frontends can display options to enable them or not. -To do this, add an entry to `pkg/routes/api/v1/info.go`. +To do this, add an entry to the `AvailableMigrators` field in `pkg/routes/api/v1/info.go`. diff --git a/pkg/caldav/parsing.go b/pkg/caldav/parsing.go index 0b566743..ead9ebcc 100644 --- a/pkg/caldav/parsing.go +++ b/pkg/caldav/parsing.go @@ -26,7 +26,7 @@ import ( "github.com/laurent22/ical-go" ) -func GetCaldavTodosForTasks(list *models.List, listTasks []*models.Task) string { +func GetCaldavTodosForTasks(list *models.ListWithTasksAndBuckets, listTasks []*models.TaskWithComments) string { // Make caldav todos from Vikunja todos var caldavtodos []*Todo diff --git a/pkg/files/files.go b/pkg/files/files.go index 58986647..08beb977 100644 --- a/pkg/files/files.go +++ b/pkg/files/files.go @@ -22,6 +22,8 @@ import ( "strconv" "time" + "xorm.io/xorm" + "code.vikunja.io/api/pkg/config" "code.vikunja.io/api/pkg/db" "code.vikunja.io/api/pkg/log" @@ -77,7 +79,18 @@ func Create(f io.Reader, realname string, realsize uint64, a web.Auth) (file *Fi // CreateWithMime creates a new file from an FileHeader and sets its mime type func CreateWithMime(f io.Reader, realname string, realsize uint64, a web.Auth, mime string) (file *File, err error) { + s := db.NewSession() + defer s.Close() + file, err = CreateWithMimeAndSession(s, f, realname, realsize, a, mime) + if err != nil { + _ = s.Rollback() + return + } + return +} + +func CreateWithMimeAndSession(s *xorm.Session, f io.Reader, realname string, realsize uint64, a web.Auth, mime string) (file *File, err error) { // Get and parse the configured file size var maxSize datasize.ByteSize err = maxSize.UnmarshalText([]byte(config.FilesMaxSize.GetString())) @@ -96,21 +109,13 @@ func CreateWithMime(f io.Reader, realname string, realsize uint64, a web.Auth, m Mime: mime, } - s := db.NewSession() - defer s.Close() - _, err = s.Insert(file) if err != nil { - _ = s.Rollback() return } // Save the file to storage with its new ID as path err = file.Save(f) - if err != nil { - _ = s.Rollback() - return - } return } diff --git a/pkg/initialize/init.go b/pkg/initialize/init.go index 32036143..b2b0d3b5 100644 --- a/pkg/initialize/init.go +++ b/pkg/initialize/init.go @@ -97,6 +97,7 @@ func FullInit() { user.RegisterTokenCleanupCron() user.RegisterDeletionNotificationCron() models.RegisterUserDeletionCron() + models.RegisterOldExportCleanupCron() // Start processing events go func() { diff --git a/pkg/migration/20210829194722.go b/pkg/migration/20210829194722.go new file mode 100644 index 00000000..111ce704 --- /dev/null +++ b/pkg/migration/20210829194722.go @@ -0,0 +1,43 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2021 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public Licensee as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public Licensee for more details. +// +// You should have received a copy of the GNU Affero General Public Licensee +// along with this program. If not, see . + +package migration + +import ( + "src.techknowlogick.com/xormigrate" + "xorm.io/xorm" +) + +type users20210829194722 struct { + ExportFileID int64 `xorm:"bigint null" json:"-"` +} + +func (users20210829194722) TableName() string { + return "users" +} + +func init() { + migrations = append(migrations, &xormigrate.Migration{ + ID: "20210829194722", + Description: "Add data export file id to users", + Migrate: func(tx *xorm.Engine) error { + return tx.Sync2(users20210829194722{}) + }, + Rollback: func(tx *xorm.Engine) error { + return nil + }, + }) +} diff --git a/pkg/models/events.go b/pkg/models/events.go index f1651b10..909591f5 100644 --- a/pkg/models/events.go +++ b/pkg/models/events.go @@ -21,6 +21,16 @@ import ( "code.vikunja.io/web" ) +// DataExportRequestEvent represents a DataExportRequestEvent event +type DataExportRequestEvent struct { + User *user.User +} + +// Name defines the name for DataExportRequestEvent +func (t *DataExportRequestEvent) Name() string { + return "user.export.request" +} + ///////////////// // Task Events // ///////////////// @@ -257,3 +267,13 @@ type TeamDeletedEvent struct { func (t *TeamDeletedEvent) Name() string { return "team.deleted" } + +// UserDataExportRequestedEvent represents a UserDataExportRequestedEvent event +type UserDataExportRequestedEvent struct { + User *user.User +} + +// Name defines the name for UserDataExportRequestedEvent +func (t *UserDataExportRequestedEvent) Name() string { + return "user.export.requested" +} diff --git a/pkg/models/export.go b/pkg/models/export.go new file mode 100644 index 00000000..2b600c37 --- /dev/null +++ b/pkg/models/export.go @@ -0,0 +1,358 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2021 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public Licensee as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public Licensee for more details. +// +// You should have received a copy of the GNU Affero General Public Licensee +// along with this program. If not, see . + +package models + +import ( + "archive/zip" + "encoding/json" + "fmt" + "io" + "os" + "strconv" + "time" + + "code.vikunja.io/api/pkg/config" + "code.vikunja.io/api/pkg/cron" + "code.vikunja.io/api/pkg/db" + "code.vikunja.io/api/pkg/files" + "code.vikunja.io/api/pkg/log" + "code.vikunja.io/api/pkg/notifications" + "code.vikunja.io/api/pkg/user" + "code.vikunja.io/api/pkg/utils" + "code.vikunja.io/api/pkg/version" + "xorm.io/xorm" +) + +func ExportUserData(s *xorm.Session, u *user.User) (err error) { + exportDir := config.ServiceRootpath.GetString() + "/files/user-export-tmp/" + err = os.MkdirAll(exportDir, 0700) + if err != nil { + return err + } + + tmpFilename := exportDir + strconv.FormatInt(u.ID, 10) + "_" + time.Now().Format("2006-01-02_15-03-05") + ".zip" + + // Open zip + dumpFile, err := os.Create(tmpFilename) + if err != nil { + return fmt.Errorf("error opening dump file: %s", err) + } + defer dumpFile.Close() + + dumpWriter := zip.NewWriter(dumpFile) + defer dumpWriter.Close() + + // Get the data + err = exportListsAndTasks(s, u, dumpWriter) + if err != nil { + return err + } + // Task attachment files + err = exportTaskAttachments(s, u, dumpWriter) + if err != nil { + return err + } + // Saved filters + err = exportSavedFilters(s, u, dumpWriter) + if err != nil { + return err + } + // Background files + err = exportListBackgrounds(s, u, dumpWriter) + if err != nil { + return err + } + // Vikunja Version + err = utils.WriteBytesToZip("VERSION", []byte(version.Version), dumpWriter) + if err != nil { + return err + } + + // If we reuse the same file again, saving it as a file in Vikunja will save it as a file with 0 bytes in size. + // Closing and reopening does work. + dumpWriter.Close() + dumpFile.Close() + + exported, err := os.Open(tmpFilename) + if err != nil { + return err + } + + stat, err := exported.Stat() + if err != nil { + return err + } + + exportFile, err := files.CreateWithMimeAndSession(s, exported, tmpFilename, uint64(stat.Size()), u, "application/zip") + if err != nil { + return err + } + + // Save the file id with the user + u.ExportFileID = exportFile.ID + _, err = s.Cols("export_file_id").Update(u) + if err != nil { + return + } + + // Remove the old file + err = os.Remove(exported.Name()) + if err != nil { + return err + } + + // Send a notification + return notifications.Notify(u, &DataExportReadyNotification{ + User: u, + }) +} + +func exportListsAndTasks(s *xorm.Session, u *user.User, wr *zip.Writer) (err error) { + + namspaces, _, _, err := (&Namespace{}).ReadAll(s, u, "", -1, 0) + if err != nil { + return err + } + + namespaceIDs := []int64{} + namespaces := []*NamespaceWithListsAndTasks{} + for _, n := range namspaces.([]*NamespaceWithLists) { + if n.ID < 1 { + // Don't include filters + continue + } + + nn := &NamespaceWithListsAndTasks{ + Namespace: n.Namespace, + Lists: []*ListWithTasksAndBuckets{}, + } + + for _, l := range n.Lists { + nn.Lists = append(nn.Lists, &ListWithTasksAndBuckets{ + List: *l, + BackgroundFileID: l.BackgroundFileID, + Tasks: []*TaskWithComments{}, + }) + } + + namespaceIDs = append(namespaceIDs, n.ID) + namespaces = append(namespaces, nn) + } + + if len(namespaceIDs) == 0 { + return nil + } + + // Get all lists + lists, err := getListsForNamespaces(s, namespaceIDs, true) + if err != nil { + return err + } + + tasks, _, _, err := getTasksForLists(s, lists, u, &taskOptions{ + page: 0, + perPage: -1, + }) + if err != nil { + return err + } + + listMap := make(map[int64]*ListWithTasksAndBuckets) + listIDs := []int64{} + for _, n := range namespaces { + for _, l := range n.Lists { + listMap[l.ID] = l + listIDs = append(listIDs, l.ID) + } + } + + taskMap := make(map[int64]*TaskWithComments, len(tasks)) + for _, t := range tasks { + taskMap[t.ID] = &TaskWithComments{ + Task: *t, + } + listMap[t.ListID].Tasks = append(listMap[t.ListID].Tasks, taskMap[t.ID]) + } + + comments := []*TaskComment{} + err = s. + Join("LEFT", "tasks", "tasks.id = task_comments.task_id"). + In("tasks.list_id", listIDs). + Find(&comments) + if err != nil { + return + } + + for _, c := range comments { + taskMap[c.TaskID].Comments = append(taskMap[c.TaskID].Comments, c) + } + + buckets := []*Bucket{} + err = s.In("list_id", listIDs).Find(&buckets) + if err != nil { + return + } + + for _, b := range buckets { + listMap[b.ListID].Buckets = append(listMap[b.ListID].Buckets, b) + } + + data, err := json.Marshal(namespaces) + if err != nil { + return err + } + + return utils.WriteBytesToZip("data.json", data, wr) +} + +func exportTaskAttachments(s *xorm.Session, u *user.User, wr *zip.Writer) (err error) { + lists, _, _, err := getRawListsForUser( + s, + &listOptions{ + user: u, + page: -1, + }, + ) + if err != nil { + return err + } + + tasks, _, _, err := getRawTasksForLists(s, lists, u, &taskOptions{page: -1}) + if err != nil { + return err + } + + taskIDs := []int64{} + for _, t := range tasks { + taskIDs = append(taskIDs, t.ID) + } + + tas, err := getTaskAttachmentsByTaskIDs(s, taskIDs) + if err != nil { + return err + } + + fs := make(map[int64]io.ReadCloser) + for _, ta := range tas { + if err := ta.File.LoadFileByID(); err != nil { + return err + } + fs[ta.FileID] = ta.File.File + } + + return utils.WriteFilesToZip(fs, wr) +} + +func exportSavedFilters(s *xorm.Session, u *user.User, wr *zip.Writer) (err error) { + filters, err := getSavedFiltersForUser(s, u) + if err != nil { + return err + } + + data, err := json.Marshal(filters) + if err != nil { + return err + } + + return utils.WriteBytesToZip("filters.json", data, wr) +} + +func exportListBackgrounds(s *xorm.Session, u *user.User, wr *zip.Writer) (err error) { + lists, _, _, err := getRawListsForUser( + s, + &listOptions{ + user: u, + page: -1, + }, + ) + if err != nil { + return err + } + + fs := make(map[int64]io.ReadCloser) + for _, l := range lists { + if l.BackgroundFileID == 0 { + continue + } + + bgFile := &files.File{ + ID: l.BackgroundFileID, + } + err = bgFile.LoadFileByID() + if err != nil { + return + } + + fs[l.BackgroundFileID] = bgFile.File + } + + return utils.WriteFilesToZip(fs, wr) +} + +func RegisterOldExportCleanupCron() { + const logPrefix = "[User Export Cleanup Cron] " + + err := cron.Schedule("0 * * * *", func() { + s := db.NewSession() + defer s.Close() + + users := []*user.User{} + err := s.Where("export_file_id IS NOT NULL AND export_file_id != ?", 0).Find(&users) + if err != nil { + log.Errorf(logPrefix+"Could not get users with export files: %s", err) + return + } + + fileIDs := []int64{} + for _, u := range users { + fileIDs = append(fileIDs, u.ExportFileID) + } + + fs := []*files.File{} + err = s.Where("created < ?", time.Now().Add(-time.Hour*24*7)).In("id", fileIDs).Find(&fs) + if err != nil { + log.Errorf(logPrefix+"Could not get users with export files: %s", err) + return + } + + if len(fs) == 0 { + return + } + + log.Debugf(logPrefix+"Removing %d old user data exports...", len(fs)) + + for _, f := range fs { + err = f.Delete() + if err != nil { + log.Errorf(logPrefix+"Could not remove user export file %d: %s", f.ID, err) + return + } + } + + _, err = s.In("export_file_id", fileIDs).Cols("export_file_id").Update(&user.User{}) + if err != nil { + log.Errorf(logPrefix+"Could not update user export file state: %s", err) + return + } + + log.Debugf(logPrefix+"Removed %d old user data exports...", len(fs)) + + }) + if err != nil { + log.Fatalf("Could not old export cleanup cron: %s", err) + } +} diff --git a/pkg/models/list.go b/pkg/models/list.go index a3ac2c97..fc899e0f 100644 --- a/pkg/models/list.go +++ b/pkg/models/list.go @@ -51,12 +51,6 @@ type List struct { // The user who created this list. Owner *user.User `xorm:"-" json:"owner" valid:"-"` - // An array of tasks which belong to the list. - // Deprecated: you should use the dedicated task list endpoint because it has support for pagination and filtering - Tasks []*Task `xorm:"-" json:"-"` - - // Only used for migration. - Buckets []*Bucket `xorm:"-" json:"-"` // Whether or not a list is archived. IsArchived bool `xorm:"not null default false" json:"is_archived" query:"is_archived"` @@ -85,6 +79,15 @@ type List struct { web.Rights `xorm:"-" json:"-"` } +type ListWithTasksAndBuckets struct { + List + // An array of tasks which belong to the list. + Tasks []*TaskWithComments `xorm:"-" json:"tasks"` + // Only used for migration. + Buckets []*Bucket `xorm:"-" json:"buckets"` + BackgroundFileID int64 `xorm:"null" json:"background_file_id"` +} + // TableName returns a better name for the lists table func (l *List) TableName() string { return "lists" diff --git a/pkg/models/listeners.go b/pkg/models/listeners.go index 9b5c8c1d..b16b52f5 100644 --- a/pkg/models/listeners.go +++ b/pkg/models/listeners.go @@ -50,6 +50,7 @@ func RegisterListeners() { events.RegisterListener((&TaskCommentUpdatedEvent{}).Name(), &HandleTaskCommentEditMentions{}) events.RegisterListener((&TaskCreatedEvent{}).Name(), &HandleTaskCreateMentions{}) events.RegisterListener((&TaskUpdatedEvent{}).Name(), &HandleTaskUpdatedMentions{}) + events.RegisterListener((&UserDataExportRequestedEvent{}).Name(), &HandleUserDataExport{}) } ////// @@ -562,3 +563,41 @@ func (s *SendTeamMemberAddedNotification) Handle(msg *message.Message) (err erro Team: event.Team, }) } + +// HandleUserDataExport represents a listener +type HandleUserDataExport struct { +} + +// Name defines the name for the HandleUserDataExport listener +func (s *HandleUserDataExport) Name() string { + return "handle.user.data.export" +} + +// Handle is executed when the event HandleUserDataExport listens on is fired +func (s *HandleUserDataExport) Handle(msg *message.Message) (err error) { + event := &UserDataExportRequestedEvent{} + err = json.Unmarshal(msg.Payload, event) + if err != nil { + return err + } + + log.Debugf("Starting to export user data for user %d...", event.User.ID) + + sess := db.NewSession() + defer sess.Close() + err = sess.Begin() + if err != nil { + return + } + + err = ExportUserData(sess, event.User) + if err != nil { + _ = sess.Rollback() + return + } + + log.Debugf("Done exporting user data for user %d...", event.User.ID) + + err = sess.Commit() + return err +} diff --git a/pkg/models/namespace.go b/pkg/models/namespace.go index 84a8ad93..f1d465b0 100644 --- a/pkg/models/namespace.go +++ b/pkg/models/namespace.go @@ -187,6 +187,11 @@ type NamespaceWithLists struct { Lists []*List `xorm:"-" json:"lists"` } +type NamespaceWithListsAndTasks struct { + Namespace + Lists []*ListWithTasksAndBuckets `xorm:"-" json:"lists"` +} + func makeNamespaceSlice(namespaces map[int64]*NamespaceWithLists, userMap map[int64]*user.User, subscriptions map[int64]*Subscription) []*NamespaceWithLists { all := make([]*NamespaceWithLists, 0, len(namespaces)) for _, n := range namespaces { diff --git a/pkg/models/notifications.go b/pkg/models/notifications.go index 2e649ffa..71a30d3f 100644 --- a/pkg/models/notifications.go +++ b/pkg/models/notifications.go @@ -302,3 +302,29 @@ func (n *UserMentionedInTaskNotification) ToDB() interface{} { func (n *UserMentionedInTaskNotification) Name() string { return "task.mentioned" } + +// DataExportReadyNotification represents a DataExportReadyNotification notification +type DataExportReadyNotification struct { + User *user.User `json:"user"` +} + +// ToMail returns the mail notification for DataExportReadyNotification +func (n *DataExportReadyNotification) ToMail() *notifications.Mail { + return notifications.NewMail(). + Subject("Your Vikunja Data Export is ready"). + Greeting("Hi "+n.User.GetName()+","). + Line("Your Vikunja Data Export is ready for you to download. Click the button below to download it:"). + Action("Download", config.ServiceFrontendurl.GetString()+"user/export/download"). + Line("The download will be available for the next 7 days."). + Line("Have a nice day!") +} + +// ToDB returns the DataExportReadyNotification notification in a format which can be saved in the db +func (n *DataExportReadyNotification) ToDB() interface{} { + return nil +} + +// Name returns the name of the notification +func (n *DataExportReadyNotification) Name() string { + return "data.export.ready" +} diff --git a/pkg/models/tasks.go b/pkg/models/tasks.go index 840129ea..f32c7ecd 100644 --- a/pkg/models/tasks.go +++ b/pkg/models/tasks.go @@ -129,6 +129,11 @@ type Task struct { web.Rights `xorm:"-" json:"-"` } +type TaskWithComments struct { + Task + Comments []*TaskComment `xorm:"-" json:"comments"` +} + // TableName returns the table name for listtasks func (Task) TableName() string { return "tasks" diff --git a/pkg/modules/dump/dump.go b/pkg/modules/dump/dump.go index b5862fec..48f85256 100644 --- a/pkg/modules/dump/dump.go +++ b/pkg/modules/dump/dump.go @@ -21,19 +21,15 @@ import ( "fmt" "io" "os" - "strconv" "code.vikunja.io/api/pkg/db" "code.vikunja.io/api/pkg/files" "code.vikunja.io/api/pkg/log" + "code.vikunja.io/api/pkg/utils" "code.vikunja.io/api/pkg/version" "github.com/spf13/viper" ) -// Change to deflate to gain better compression -// see http://golang.org/pkg/archive/zip/#pkg-constants -const compressionUsed = zip.Deflate - // Dump creates a zip file with all vikunja files at filename func Dump(filename string) error { dumpFile, err := os.Create(filename) @@ -55,7 +51,7 @@ func Dump(filename string) error { // Version log.Info("Start dumping version file...") - err = writeBytesToZip("VERSION", []byte(version.Version), dumpWriter) + err = utils.WriteBytesToZip("VERSION", []byte(version.Version), dumpWriter) if err != nil { return fmt.Errorf("error saving version: %s", err) } @@ -68,7 +64,7 @@ func Dump(filename string) error { return fmt.Errorf("error saving database data: %s", err) } for t, d := range data { - err = writeBytesToZip("database/"+t+".json", d, dumpWriter) + err = utils.WriteBytesToZip("database/"+t+".json", d, dumpWriter) if err != nil { return fmt.Errorf("error writing database table %s: %s", t, err) } @@ -81,21 +77,12 @@ func Dump(filename string) error { if err != nil { return fmt.Errorf("error saving file: %s", err) } - for fid, file := range allFiles { - header := &zip.FileHeader{ - Name: "files/" + strconv.FormatInt(fid, 10), - Method: compressionUsed, - } - w, err := dumpWriter.CreateHeader(header) - if err != nil { - return err - } - _, err = io.Copy(w, file) - if err != nil { - return fmt.Errorf("error writing file %d: %s", fid, err) - } - _ = file.Close() + + err = utils.WriteFilesToZip(allFiles, dumpWriter) + if err != nil { + return err } + log.Infof("Dumped files") log.Info("Done creating dump") @@ -123,7 +110,7 @@ func writeFileToZip(filename string, writer *zip.Writer) error { } header.Name = info.Name() - header.Method = compressionUsed + header.Method = utils.CompressionUsed w, err := writer.CreateHeader(header) if err != nil { @@ -132,16 +119,3 @@ func writeFileToZip(filename string, writer *zip.Writer) error { _, err = io.Copy(w, fileToZip) return err } - -func writeBytesToZip(filename string, data []byte, writer *zip.Writer) (err error) { - header := &zip.FileHeader{ - Name: filename, - Method: compressionUsed, - } - w, err := writer.CreateHeader(header) - if err != nil { - return err - } - _, err = w.Write(data) - return -} diff --git a/pkg/modules/dump/restore.go b/pkg/modules/dump/restore.go index 4de3c5fc..b316167a 100644 --- a/pkg/modules/dump/restore.go +++ b/pkg/modules/dump/restore.go @@ -33,6 +33,7 @@ import ( "code.vikunja.io/api/pkg/initialize" "code.vikunja.io/api/pkg/log" "code.vikunja.io/api/pkg/migration" + "src.techknowlogick.com/xormigrate" ) @@ -194,6 +195,7 @@ func Restore(filename string) error { /////// // Done log.Infof("Done restoring dump.") + log.Infof("Restart Vikunja to make sure the new configuration file is applied.") return nil } diff --git a/pkg/modules/migration/create_from_structure.go b/pkg/modules/migration/create_from_structure.go index 148c8aee..8be8304c 100644 --- a/pkg/modules/migration/create_from_structure.go +++ b/pkg/modules/migration/create_from_structure.go @@ -31,7 +31,7 @@ import ( // InsertFromStructure takes a fully nested Vikunja data structure and a user and then creates everything for this user // (Namespaces, tasks, etc. Even attachments and relations.) -func InsertFromStructure(str []*models.NamespaceWithLists, user *user.User) (err error) { +func InsertFromStructure(str []*models.NamespaceWithListsAndTasks, user *user.User) (err error) { s := db.NewSession() defer s.Close() @@ -45,7 +45,7 @@ func InsertFromStructure(str []*models.NamespaceWithLists, user *user.User) (err return s.Commit() } -func insertFromStructure(s *xorm.Session, str []*models.NamespaceWithLists, user *user.User) (err error) { +func insertFromStructure(s *xorm.Session, str []*models.NamespaceWithListsAndTasks, user *user.User) (err error) { log.Debugf("[creating structure] Creating %d namespaces", len(str)) @@ -129,7 +129,7 @@ func insertFromStructure(s *xorm.Session, str []*models.NamespaceWithLists, user // Create all tasks for _, t := range tasks { - setBucketOrDefault(t) + setBucketOrDefault(&t.Task) t.ListID = l.ID err = t.Create(s, user) @@ -221,6 +221,15 @@ func insertFromStructure(s *xorm.Session, str []*models.NamespaceWithLists, user } log.Debugf("[creating structure] Associated task %d with label %d", t.ID, lb.ID) } + + for _, comment := range t.Comments { + comment.TaskID = t.ID + err = comment.Create(s, user) + if err != nil { + return + } + log.Debugf("[creating structure] Created new comment %d", comment.ID) + } } // All tasks brought their own bucket with them, therefore the newly created default bucket is just extra space diff --git a/pkg/modules/migration/create_from_structure_test.go b/pkg/modules/migration/create_from_structure_test.go index bdd4d6aa..1b3eb5d4 100644 --- a/pkg/modules/migration/create_from_structure_test.go +++ b/pkg/modules/migration/create_from_structure_test.go @@ -32,79 +32,95 @@ func TestInsertFromStructure(t *testing.T) { } t.Run("normal", func(t *testing.T) { db.LoadAndAssertFixtures(t) - testStructure := []*models.NamespaceWithLists{ + testStructure := []*models.NamespaceWithListsAndTasks{ { Namespace: models.Namespace{ Title: "Test1", Description: "Lorem Ipsum", }, - Lists: []*models.List{ + Lists: []*models.ListWithTasksAndBuckets{ { - Title: "Testlist1", - Description: "Something", + List: models.List{ + Title: "Testlist1", + Description: "Something", + }, Buckets: []*models.Bucket{ { ID: 1234, Title: "Test Bucket", }, }, - Tasks: []*models.Task{ + Tasks: []*models.TaskWithComments{ { - Title: "Task1", - Description: "Lorem", + Task: models.Task{ + Title: "Task1", + Description: "Lorem", + }, }, { - Title: "Task with related tasks", - RelatedTasks: map[models.RelationKind][]*models.Task{ - models.RelationKindSubtask: { + Task: models.Task{ + Title: "Task with related tasks", + RelatedTasks: map[models.RelationKind][]*models.Task{ + models.RelationKindSubtask: { + { + Title: "Related to task with related task", + Description: "As subtask", + }, + }, + }, + }, + }, + { + Task: models.Task{ + Title: "Task with attachments", + Attachments: []*models.TaskAttachment{ { - Title: "Related to task with related task", - Description: "As subtask", + File: &files.File{ + Name: "testfile", + Size: 4, + FileContent: []byte{1, 2, 3, 4}, + }, }, }, }, }, { - Title: "Task with attachments", - Attachments: []*models.TaskAttachment{ - { - File: &files.File{ - Name: "testfile", - Size: 4, - FileContent: []byte{1, 2, 3, 4}, + Task: models.Task{ + Title: "Task with labels", + Labels: []*models.Label{ + { + Title: "Label1", + HexColor: "ff00ff", + }, + { + Title: "Label2", + HexColor: "ff00ff", }, }, }, }, { - Title: "Task with labels", - Labels: []*models.Label{ - { - Title: "Label1", - HexColor: "ff00ff", - }, - { - Title: "Label2", - HexColor: "ff00ff", + Task: models.Task{ + Title: "Task with same label", + Labels: []*models.Label{ + { + Title: "Label1", + HexColor: "ff00ff", + }, }, }, }, { - Title: "Task with same label", - Labels: []*models.Label{ - { - Title: "Label1", - HexColor: "ff00ff", - }, + Task: models.Task{ + Title: "Task in a bucket", + BucketID: 1234, }, }, { - Title: "Task in a bucket", - BucketID: 1234, - }, - { - Title: "Task in a nonexisting bucket", - BucketID: 1111, + Task: models.Task{ + Title: "Task in a nonexisting bucket", + BucketID: 1111, + }, }, }, }, diff --git a/pkg/modules/migration/handler/common.go b/pkg/modules/migration/handler/common.go new file mode 100644 index 00000000..2dfe0444 --- /dev/null +++ b/pkg/modules/migration/handler/common.go @@ -0,0 +1,40 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2021 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public Licensee as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public Licensee for more details. +// +// You should have received a copy of the GNU Affero General Public Licensee +// along with this program. If not, see . + +package handler + +import ( + "net/http" + + "code.vikunja.io/api/pkg/modules/migration" + user2 "code.vikunja.io/api/pkg/user" + "code.vikunja.io/web/handler" + "github.com/labstack/echo/v4" +) + +func status(ms migration.MigratorName, c echo.Context) error { + user, err := user2.GetCurrentUser(c) + if err != nil { + return handler.HandleHTTPError(err, c) + } + + status, err := migration.GetMigrationStatus(ms, user) + if err != nil { + return handler.HandleHTTPError(err, c) + } + + return c.JSON(http.StatusOK, status) +} diff --git a/pkg/modules/migration/handler/handler.go b/pkg/modules/migration/handler/handler.go index 158f680a..6ff70e9c 100644 --- a/pkg/modules/migration/handler/handler.go +++ b/pkg/modules/migration/handler/handler.go @@ -84,15 +84,5 @@ func (mw *MigrationWeb) Migrate(c echo.Context) error { func (mw *MigrationWeb) Status(c echo.Context) error { ms := mw.MigrationStruct() - user, err := user2.GetCurrentUser(c) - if err != nil { - return handler.HandleHTTPError(err, c) - } - - status, err := migration.GetMigrationStatus(ms, user) - if err != nil { - return handler.HandleHTTPError(err, c) - } - - return c.JSON(http.StatusOK, status) + return status(ms, c) } diff --git a/pkg/modules/migration/handler/handler_file.go b/pkg/modules/migration/handler/handler_file.go new file mode 100644 index 00000000..427c0ba5 --- /dev/null +++ b/pkg/modules/migration/handler/handler_file.go @@ -0,0 +1,79 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2021 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public Licensee as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public Licensee for more details. +// +// You should have received a copy of the GNU Affero General Public Licensee +// along with this program. If not, see . + +package handler + +import ( + "net/http" + + "code.vikunja.io/api/pkg/models" + "code.vikunja.io/api/pkg/modules/migration" + user2 "code.vikunja.io/api/pkg/user" + "code.vikunja.io/web/handler" + "github.com/labstack/echo/v4" +) + +type FileMigratorWeb struct { + MigrationStruct func() migration.FileMigrator +} + +// RegisterRoutes registers all routes for migration +func (fw *FileMigratorWeb) RegisterRoutes(g *echo.Group) { + ms := fw.MigrationStruct() + g.GET("/"+ms.Name()+"/status", fw.Status) + g.PUT("/"+ms.Name()+"/migrate", fw.Migrate) +} + +// Migrate calls the migration method +func (fw *FileMigratorWeb) Migrate(c echo.Context) error { + ms := fw.MigrationStruct() + + // Get the user from context + user, err := user2.GetCurrentUser(c) + if err != nil { + return handler.HandleHTTPError(err, c) + } + + file, err := c.FormFile("import") + if err != nil { + return err + } + src, err := file.Open() + if err != nil { + return err + } + defer src.Close() + + // Do the migration + err = ms.Migrate(user, src, file.Size) + if err != nil { + return handler.HandleHTTPError(err, c) + } + + err = migration.SetMigrationStatus(ms, user) + if err != nil { + return handler.HandleHTTPError(err, c) + } + + return c.JSON(http.StatusOK, models.Message{Message: "Everything was migrated successfully."}) +} + +// Status returns whether or not a user has already done this migration +func (fw *FileMigratorWeb) Status(c echo.Context) error { + ms := fw.MigrationStruct() + + return status(ms, c) +} diff --git a/pkg/modules/migration/microsoft-todo/microsoft_todo.go b/pkg/modules/migration/microsoft-todo/microsoft_todo.go index cba69877..64d20e41 100644 --- a/pkg/modules/migration/microsoft-todo/microsoft_todo.go +++ b/pkg/modules/migration/microsoft-todo/microsoft_todo.go @@ -243,15 +243,15 @@ func getMicrosoftTodoData(token string) (microsoftTodoData []*list, err error) { return } -func convertMicrosoftTodoData(todoData []*list) (vikunjsStructure []*models.NamespaceWithLists, err error) { +func convertMicrosoftTodoData(todoData []*list) (vikunjsStructure []*models.NamespaceWithListsAndTasks, err error) { // One namespace with all lists - vikunjsStructure = []*models.NamespaceWithLists{ + vikunjsStructure = []*models.NamespaceWithListsAndTasks{ { Namespace: models.Namespace{ Title: "Migrated from Microsoft Todo", }, - Lists: []*models.List{}, + Lists: []*models.ListWithTasksAndBuckets{}, }, } @@ -262,8 +262,10 @@ func convertMicrosoftTodoData(todoData []*list) (vikunjsStructure []*models.Name log.Debugf("[Microsoft Todo Migration] Converting list %s", l.ID) // Lists only with title - list := &models.List{ - Title: l.DisplayName, + list := &models.ListWithTasksAndBuckets{ + List: models.List{ + Title: l.DisplayName, + }, } log.Debugf("[Microsoft Todo Migration] Converting %d tasks", len(l.Tasks)) @@ -340,7 +342,7 @@ func convertMicrosoftTodoData(todoData []*list) (vikunjsStructure []*models.Name } } - list.Tasks = append(list.Tasks, task) + list.Tasks = append(list.Tasks, &models.TaskWithComments{Task: *task}) log.Debugf("[Microsoft Todo Migration] Done converted %d tasks", len(l.Tasks)) } diff --git a/pkg/modules/migration/microsoft-todo/microsoft_todo_test.go b/pkg/modules/migration/microsoft-todo/microsoft_todo_test.go index a0d229bd..a9951156 100644 --- a/pkg/modules/migration/microsoft-todo/microsoft_todo_test.go +++ b/pkg/modules/migration/microsoft-todo/microsoft_todo_test.go @@ -102,57 +102,79 @@ func TestConverting(t *testing.T) { }, } - expectedHierachie := []*models.NamespaceWithLists{ + expectedHierachie := []*models.NamespaceWithListsAndTasks{ { Namespace: models.Namespace{ Title: "Migrated from Microsoft Todo", }, - Lists: []*models.List{ + Lists: []*models.ListWithTasksAndBuckets{ { - Title: "List 1", - Tasks: []*models.Task{ + List: models.List{ + Title: "List 1", + }, + Tasks: []*models.TaskWithComments{ { - Title: "Task 1", - Description: "This is a description", - }, - { - Title: "Task 2", - Done: true, - DoneAt: testtimeTime, - }, - { - Title: "Task 3", - Priority: 1, - }, - { - Title: "Task 4", - Priority: 3, - }, - { - Title: "Task 5", - Reminders: []time.Time{ - testtimeTime, + Task: models.Task{ + Title: "Task 1", + Description: "This is a description", }, }, { - Title: "Task 6", - DueDate: testtimeTime, + Task: models.Task{ + Title: "Task 2", + Done: true, + DoneAt: testtimeTime, + }, }, { - Title: "Task 7", - DueDate: testtimeTime, - RepeatAfter: 60 * 60 * 24 * 7, // The amount of seconds in a week + Task: models.Task{ + Title: "Task 3", + Priority: 1, + }, + }, + { + Task: models.Task{ + Title: "Task 4", + Priority: 3, + }, + }, + { + Task: models.Task{ + Title: "Task 5", + Reminders: []time.Time{ + testtimeTime, + }, + }, + }, + { + Task: models.Task{ + Title: "Task 6", + DueDate: testtimeTime, + }, + }, + { + Task: models.Task{ + Title: "Task 7", + DueDate: testtimeTime, + RepeatAfter: 60 * 60 * 24 * 7, // The amount of seconds in a week + }, }, }, }, { - Title: "List 2", - Tasks: []*models.Task{ + List: models.List{ + Title: "List 2", + }, + Tasks: []*models.TaskWithComments{ { - Title: "Task 1", + Task: models.Task{ + Title: "Task 1", + }, }, { - Title: "Task 2", + Task: models.Task{ + Title: "Task 2", + }, }, }, }, diff --git a/pkg/modules/migration/migration_status.go b/pkg/modules/migration/migration_status.go index c0421a0e..e23a502e 100644 --- a/pkg/modules/migration/migration_status.go +++ b/pkg/modules/migration/migration_status.go @@ -37,7 +37,7 @@ func (s *Status) TableName() string { } // SetMigrationStatus sets the migration status for a user -func SetMigrationStatus(m Migrator, u *user.User) (err error) { +func SetMigrationStatus(m MigratorName, u *user.User) (err error) { s := db.NewSession() defer s.Close() @@ -50,7 +50,7 @@ func SetMigrationStatus(m Migrator, u *user.User) (err error) { } // GetMigrationStatus returns the migration status for a migration and a user -func GetMigrationStatus(m Migrator, u *user.User) (status *Status, err error) { +func GetMigrationStatus(m MigratorName, u *user.User) (status *Status, err error) { s := db.NewSession() defer s.Close() diff --git a/pkg/modules/migration/migrator.go b/pkg/modules/migration/migrator.go index 285654c2..c7ca2d31 100644 --- a/pkg/modules/migration/migrator.go +++ b/pkg/modules/migration/migrator.go @@ -17,11 +17,20 @@ package migration import ( + "io" + "code.vikunja.io/api/pkg/user" ) +type MigratorName interface { + // Name holds the name of the migration. + // This is used to show the name to users and to keep track of users who already migrated. + Name() string +} + // Migrator is the basic migrator interface which is shared among all migrators type Migrator interface { + MigratorName // Migrate is the interface used to migrate a user's tasks from another platform to vikunja. // The user object is the user who's tasks will be migrated. Migrate(user *user.User) error @@ -29,7 +38,12 @@ type Migrator interface { // The use case for this are Oauth flows, where the server token should remain hidden and not // known to the frontend. AuthURL() string - // Title holds the name of the migration. - // This is used to show the name to users and to keep track of users who already migrated. - Name() string +} + +// FileMigrator handles importing Vikunja data from a file. The implementation of it determines the format. +type FileMigrator interface { + MigratorName + // Migrate is the interface used to migrate a user's tasks, list and other things from a file to vikunja. + // The user object is the user who's tasks will be migrated. + Migrate(user *user.User, file io.ReaderAt, size int64) error } diff --git a/pkg/modules/migration/todoist/todoist.go b/pkg/modules/migration/todoist/todoist.go index 3fe8681e..e56dfd2e 100644 --- a/pkg/modules/migration/todoist/todoist.go +++ b/pkg/modules/migration/todoist/todoist.go @@ -252,28 +252,30 @@ func parseDate(dateString string) (date time.Time, err error) { return date, err } -func convertTodoistToVikunja(sync *sync, doneItems map[int64]*doneItem) (fullVikunjaHierachie []*models.NamespaceWithLists, err error) { +func convertTodoistToVikunja(sync *sync, doneItems map[int64]*doneItem) (fullVikunjaHierachie []*models.NamespaceWithListsAndTasks, err error) { - newNamespace := &models.NamespaceWithLists{ + newNamespace := &models.NamespaceWithListsAndTasks{ Namespace: models.Namespace{ Title: "Migrated from todoist", }, } // A map for all vikunja lists with the project id they're coming from as key - lists := make(map[int64]*models.List, len(sync.Projects)) + lists := make(map[int64]*models.ListWithTasksAndBuckets, len(sync.Projects)) // A map for all vikunja tasks with the todoist task id as key to find them easily and add more data - tasks := make(map[int64]*models.Task, len(sync.Items)) + tasks := make(map[int64]*models.TaskWithComments, len(sync.Items)) // A map for all vikunja labels with the todoist id as key to find them easier labels := make(map[int64]*models.Label, len(sync.Labels)) for _, p := range sync.Projects { - list := &models.List{ - Title: p.Name, - HexColor: todoistColors[p.Color], - IsArchived: p.IsArchived == 1, + list := &models.ListWithTasksAndBuckets{ + List: models.List{ + Title: p.Name, + HexColor: todoistColors[p.Color], + IsArchived: p.IsArchived == 1, + }, } lists[p.ID] = list @@ -305,11 +307,13 @@ func convertTodoistToVikunja(sync *sync, doneItems map[int64]*doneItem) (fullVik } for _, i := range sync.Items { - task := &models.Task{ - Title: i.Content, - Created: i.DateAdded.In(config.GetTimeZone()), - Done: i.Checked == 1, - BucketID: i.SectionID, + task := &models.TaskWithComments{ + Task: models.Task{ + Title: i.Content, + Created: i.DateAdded.In(config.GetTimeZone()), + Done: i.Checked == 1, + BucketID: i.SectionID, + }, } // Only try to parse the task done at date if the task is actually done @@ -365,7 +369,7 @@ func convertTodoistToVikunja(sync *sync, doneItems map[int64]*doneItem) (fullVik tasks[i.ParentID].RelatedTasks = make(models.RelatedTaskMap) } - tasks[i.ParentID].RelatedTasks[models.RelationKindSubtask] = append(tasks[i.ParentID].RelatedTasks[models.RelationKindSubtask], tasks[i.ID]) + tasks[i.ParentID].RelatedTasks[models.RelationKindSubtask] = append(tasks[i.ParentID].RelatedTasks[models.RelationKindSubtask], &tasks[i.ID].Task) // Remove the task from the top level structure, otherwise it is added twice outer: @@ -449,7 +453,7 @@ func convertTodoistToVikunja(sync *sync, doneItems map[int64]*doneItem) (fullVik tasks[r.ItemID].Reminders = append(tasks[r.ItemID].Reminders, date.In(config.GetTimeZone())) } - return []*models.NamespaceWithLists{ + return []*models.NamespaceWithListsAndTasks{ newNamespace, }, err } diff --git a/pkg/modules/migration/todoist/todoist_test.go b/pkg/modules/migration/todoist/todoist_test.go index b0efae13..24fa48b0 100644 --- a/pkg/modules/migration/todoist/todoist_test.go +++ b/pkg/modules/migration/todoist/todoist_test.go @@ -375,210 +375,258 @@ func TestConvertTodoistToVikunja(t *testing.T) { }, } - expectedHierachie := []*models.NamespaceWithLists{ + expectedHierachie := []*models.NamespaceWithListsAndTasks{ { Namespace: models.Namespace{ Title: "Migrated from todoist", }, - Lists: []*models.List{ + Lists: []*models.ListWithTasksAndBuckets{ { - Title: "Project1", - Description: "Lorem Ipsum dolor sit amet\nLorem Ipsum dolor sit amet 2\nLorem Ipsum dolor sit amet 3", - HexColor: todoistColors[30], + List: models.List{ + Title: "Project1", + Description: "Lorem Ipsum dolor sit amet\nLorem Ipsum dolor sit amet 2\nLorem Ipsum dolor sit amet 3", + HexColor: todoistColors[30], + }, Buckets: []*models.Bucket{ { ID: 1234, Title: "Some Bucket", }, }, - Tasks: []*models.Task{ + Tasks: []*models.TaskWithComments{ { - Title: "Task400000000", - Description: "Lorem Ipsum dolor sit amet", - Done: false, - Created: time1, - Reminders: []time.Time{ - time.Date(2020, time.June, 15, 0, 0, 0, 0, time.UTC).In(config.GetTimeZone()), - time.Date(2020, time.June, 16, 7, 0, 0, 0, time.UTC).In(config.GetTimeZone()), + Task: models.Task{ + Title: "Task400000000", + Description: "Lorem Ipsum dolor sit amet", + Done: false, + Created: time1, + Reminders: []time.Time{ + time.Date(2020, time.June, 15, 0, 0, 0, 0, time.UTC).In(config.GetTimeZone()), + time.Date(2020, time.June, 16, 7, 0, 0, 0, time.UTC).In(config.GetTimeZone()), + }, }, }, { - Title: "Task400000001", - Description: "Lorem Ipsum dolor sit amet", - Done: false, - Created: time1, - }, - { - Title: "Task400000002", - Done: false, - Created: time1, - Reminders: []time.Time{ - time.Date(2020, time.July, 15, 7, 0, 0, 0, time.UTC).In(config.GetTimeZone()), + Task: models.Task{ + Title: "Task400000001", + Description: "Lorem Ipsum dolor sit amet", + Done: false, + Created: time1, }, }, { - Title: "Task400000003", - Description: "Lorem Ipsum dolor sit amet", - Done: true, - DueDate: dueTime, - Created: time1, - DoneAt: time3, - Labels: vikunjaLabels, - Reminders: []time.Time{ - time.Date(2020, time.June, 15, 7, 0, 0, 0, time.UTC).In(config.GetTimeZone()), + Task: models.Task{ + Title: "Task400000002", + Done: false, + Created: time1, + Reminders: []time.Time{ + time.Date(2020, time.July, 15, 7, 0, 0, 0, time.UTC).In(config.GetTimeZone()), + }, }, }, { - Title: "Task400000004", - Done: false, - Created: time1, - Labels: vikunjaLabels, - }, - { - Title: "Task400000005", - Done: true, - DueDate: dueTime, - Created: time1, - DoneAt: time3, - Reminders: []time.Time{ - time.Date(2020, time.June, 15, 7, 0, 0, 0, time.UTC).In(config.GetTimeZone()), + Task: models.Task{ + Title: "Task400000003", + Description: "Lorem Ipsum dolor sit amet", + Done: true, + DueDate: dueTime, + Created: time1, + DoneAt: time3, + Labels: vikunjaLabels, + Reminders: []time.Time{ + time.Date(2020, time.June, 15, 7, 0, 0, 0, time.UTC).In(config.GetTimeZone()), + }, }, }, { - Title: "Task400000006", - Done: true, - DueDate: dueTime, - Created: time1, - DoneAt: time3, - RelatedTasks: map[models.RelationKind][]*models.Task{ - models.RelationKindSubtask: { + Task: models.Task{ + Title: "Task400000004", + Done: false, + Created: time1, + Labels: vikunjaLabels, + }, + }, + { + Task: models.Task{ + Title: "Task400000005", + Done: true, + DueDate: dueTime, + Created: time1, + DoneAt: time3, + Reminders: []time.Time{ + time.Date(2020, time.June, 15, 7, 0, 0, 0, time.UTC).In(config.GetTimeZone()), + }, + }, + }, + { + Task: models.Task{ + Title: "Task400000006", + Done: true, + DueDate: dueTime, + Created: time1, + DoneAt: time3, + RelatedTasks: map[models.RelationKind][]*models.Task{ + models.RelationKindSubtask: { + { + Title: "Task with parent", + Done: false, + Priority: 2, + Created: time1, + DoneAt: nilTime, + }, + }, + }, + }, + }, + { + Task: models.Task{ + Title: "Task400000106", + Done: true, + DueDate: dueTimeWithTime, + Created: time1, + DoneAt: time3, + Labels: vikunjaLabels, + }, + }, + { + Task: models.Task{ + Title: "Task400000107", + Done: true, + Created: time1, + DoneAt: time3, + }, + }, + { + Task: models.Task{ + Title: "Task400000108", + Done: true, + Created: time1, + DoneAt: time3, + }, + }, + { + Task: models.Task{ + Title: "Task400000109", + Done: true, + Created: time1, + DoneAt: time3, + BucketID: 1234, + }, + }, + }, + }, + { + List: models.List{ + Title: "Project2", + Description: "Lorem Ipsum dolor sit amet 4\nLorem Ipsum dolor sit amet 5", + HexColor: todoistColors[37], + }, + Tasks: []*models.TaskWithComments{ + { + Task: models.Task{ + Title: "Task400000007", + Done: false, + DueDate: dueTime, + Created: time1, + }, + }, + { + Task: models.Task{ + Title: "Task400000008", + Done: false, + DueDate: dueTime, + Created: time1, + }, + }, + { + Task: models.Task{ + Title: "Task400000009", + Done: false, + Created: time1, + Reminders: []time.Time{ + time.Date(2020, time.June, 15, 7, 0, 0, 0, time.UTC).In(config.GetTimeZone()), + }, + }, + }, + { + Task: models.Task{ + Title: "Task400000010", + Description: "Lorem Ipsum dolor sit amet", + Done: true, + Created: time1, + DoneAt: time3, + }, + }, + { + Task: models.Task{ + Title: "Task400000101", + Description: "Lorem Ipsum dolor sit amet", + Done: false, + Created: time1, + Attachments: []*models.TaskAttachment{ { - Title: "Task with parent", - Done: false, - Priority: 2, - Created: time1, - DoneAt: nilTime, + File: &files.File{ + Name: "file.md", + Mime: "text/plain", + Size: 12345, + Created: time1, + FileContent: exampleFile, + }, + Created: time1, }, }, }, }, { - Title: "Task400000106", - Done: true, - DueDate: dueTimeWithTime, - Created: time1, - DoneAt: time3, - Labels: vikunjaLabels, + Task: models.Task{ + Title: "Task400000102", + Done: false, + DueDate: dueTime, + Created: time1, + Labels: vikunjaLabels, + }, }, { - Title: "Task400000107", - Done: true, - Created: time1, - DoneAt: time3, + Task: models.Task{ + Title: "Task400000103", + Done: false, + Created: time1, + Labels: vikunjaLabels, + }, }, { - Title: "Task400000108", - Done: true, - Created: time1, - DoneAt: time3, + Task: models.Task{ + Title: "Task400000104", + Done: false, + Created: time1, + Labels: vikunjaLabels, + }, }, { - Title: "Task400000109", - Done: true, - Created: time1, - DoneAt: time3, - BucketID: 1234, + Task: models.Task{ + Title: "Task400000105", + Done: false, + DueDate: dueTime, + Created: time1, + Labels: vikunjaLabels, + }, }, }, }, { - Title: "Project2", - Description: "Lorem Ipsum dolor sit amet 4\nLorem Ipsum dolor sit amet 5", - HexColor: todoistColors[37], - Tasks: []*models.Task{ - { - Title: "Task400000007", - Done: false, - DueDate: dueTime, - Created: time1, - }, - { - Title: "Task400000008", - Done: false, - DueDate: dueTime, - Created: time1, - }, - { - Title: "Task400000009", - Done: false, - Created: time1, - Reminders: []time.Time{ - time.Date(2020, time.June, 15, 7, 0, 0, 0, time.UTC).In(config.GetTimeZone()), - }, - }, - { - Title: "Task400000010", - Description: "Lorem Ipsum dolor sit amet", - Done: true, - Created: time1, - DoneAt: time3, - }, - { - Title: "Task400000101", - Description: "Lorem Ipsum dolor sit amet", - Done: false, - Created: time1, - Attachments: []*models.TaskAttachment{ - { - File: &files.File{ - Name: "file.md", - Mime: "text/plain", - Size: 12345, - Created: time1, - FileContent: exampleFile, - }, - Created: time1, - }, - }, - }, - { - Title: "Task400000102", - Done: false, - DueDate: dueTime, - Created: time1, - Labels: vikunjaLabels, - }, - { - Title: "Task400000103", - Done: false, - Created: time1, - Labels: vikunjaLabels, - }, - { - Title: "Task400000104", - Done: false, - Created: time1, - Labels: vikunjaLabels, - }, - { - Title: "Task400000105", - Done: false, - DueDate: dueTime, - Created: time1, - Labels: vikunjaLabels, - }, + List: models.List{ + Title: "Project3 - Archived", + HexColor: todoistColors[37], + IsArchived: true, }, - }, - { - Title: "Project3 - Archived", - HexColor: todoistColors[37], - IsArchived: true, - Tasks: []*models.Task{ + Tasks: []*models.TaskWithComments{ { - Title: "Task400000111", - Done: true, - Created: time1, - DoneAt: time3, + Task: models.Task{ + Title: "Task400000111", + Done: true, + Created: time1, + DoneAt: time3, + }, }, }, }, diff --git a/pkg/modules/migration/trello/trello.go b/pkg/modules/migration/trello/trello.go index e6372188..57e40a3b 100644 --- a/pkg/modules/migration/trello/trello.go +++ b/pkg/modules/migration/trello/trello.go @@ -144,16 +144,16 @@ func getTrelloData(token string) (trelloData []*trello.Board, err error) { // Converts all previously obtained data from trello into the vikunja format. // `trelloData` should contain all boards with their lists and cards respectively. -func convertTrelloDataToVikunja(trelloData []*trello.Board) (fullVikunjaHierachie []*models.NamespaceWithLists, err error) { +func convertTrelloDataToVikunja(trelloData []*trello.Board) (fullVikunjaHierachie []*models.NamespaceWithListsAndTasks, err error) { log.Debugf("[Trello Migration] ") - fullVikunjaHierachie = []*models.NamespaceWithLists{ + fullVikunjaHierachie = []*models.NamespaceWithListsAndTasks{ { Namespace: models.Namespace{ Title: "Imported from Trello", }, - Lists: []*models.List{}, + Lists: []*models.ListWithTasksAndBuckets{}, }, } @@ -162,10 +162,12 @@ func convertTrelloDataToVikunja(trelloData []*trello.Board) (fullVikunjaHierachi log.Debugf("[Trello Migration] Converting %d boards to vikunja lists", len(trelloData)) for _, board := range trelloData { - list := &models.List{ - Title: board.Name, - Description: board.Desc, - IsArchived: board.Closed, + list := &models.ListWithTasksAndBuckets{ + List: models.List{ + Title: board.Name, + Description: board.Desc, + IsArchived: board.Closed, + }, } // Background @@ -269,7 +271,7 @@ func convertTrelloDataToVikunja(trelloData []*trello.Board) (fullVikunjaHierachi log.Debugf("[Trello Migration] Downloaded card attachment %s", attachment.ID) } - list.Tasks = append(list.Tasks, task) + list.Tasks = append(list.Tasks, &models.TaskWithComments{Task: *task}) } list.Buckets = append(list.Buckets, bucket) diff --git a/pkg/modules/migration/trello/trello_test.go b/pkg/modules/migration/trello/trello_test.go index 99e2170d..e007e20d 100644 --- a/pkg/modules/migration/trello/trello_test.go +++ b/pkg/modules/migration/trello/trello_test.go @@ -187,16 +187,18 @@ func TestConvertTrelloToVikunja(t *testing.T) { } trelloData[0].Prefs.BackgroundImage = "https://vikunja.io/testimage.jpg" // Using an image which we are hosting, so it'll still be up - expectedHierachie := []*models.NamespaceWithLists{ + expectedHierachie := []*models.NamespaceWithListsAndTasks{ { Namespace: models.Namespace{ Title: "Imported from Trello", }, - Lists: []*models.List{ + Lists: []*models.ListWithTasksAndBuckets{ { - Title: "TestBoard", - Description: "This is a description", - BackgroundInformation: bytes.NewBuffer(exampleFile), + List: models.List{ + Title: "TestBoard", + Description: "This is a description", + BackgroundInformation: bytes.NewBuffer(exampleFile), + }, Buckets: []*models.Bucket{ { ID: 1, @@ -207,37 +209,40 @@ func TestConvertTrelloToVikunja(t *testing.T) { Title: "Test List 2", }, }, - Tasks: []*models.Task{ + Tasks: []*models.TaskWithComments{ { - Title: "Test Card 1", - Description: "Card Description", - BucketID: 1, - KanbanPosition: 123, - DueDate: time1, - Labels: []*models.Label{ - { - Title: "Label 1", - HexColor: trelloColorMap["green"], + Task: models.Task{ + Title: "Test Card 1", + Description: "Card Description", + BucketID: 1, + KanbanPosition: 123, + DueDate: time1, + Labels: []*models.Label{ + { + Title: "Label 1", + HexColor: trelloColorMap["green"], + }, + { + Title: "Label 2", + HexColor: trelloColorMap["orange"], + }, }, - { - Title: "Label 2", - HexColor: trelloColorMap["orange"], - }, - }, - Attachments: []*models.TaskAttachment{ - { - File: &files.File{ - Name: "Testimage.jpg", - Mime: "image/jpg", - Size: uint64(len(exampleFile)), - FileContent: exampleFile, + Attachments: []*models.TaskAttachment{ + { + File: &files.File{ + Name: "Testimage.jpg", + Mime: "image/jpg", + Size: uint64(len(exampleFile)), + FileContent: exampleFile, + }, }, }, }, }, { - Title: "Test Card 2", - Description: ` + Task: models.Task{ + Title: "Test Card 2", + Description: ` ## Checklist 1 @@ -248,84 +253,105 @@ func TestConvertTrelloToVikunja(t *testing.T) { * [ ] Pending Task * [ ] Another Pending Task`, - BucketID: 1, - KanbanPosition: 124, + BucketID: 1, + KanbanPosition: 124, + }, }, { - Title: "Test Card 3", - BucketID: 1, - KanbanPosition: 126, + Task: models.Task{ + Title: "Test Card 3", + BucketID: 1, + KanbanPosition: 126, + }, }, { - Title: "Test Card 4", - BucketID: 1, - KanbanPosition: 127, - Labels: []*models.Label{ - { - Title: "Label 2", - HexColor: trelloColorMap["orange"], + Task: models.Task{ + Title: "Test Card 4", + BucketID: 1, + KanbanPosition: 127, + Labels: []*models.Label{ + { + Title: "Label 2", + HexColor: trelloColorMap["orange"], + }, }, }, }, { - Title: "Test Card 5", - BucketID: 2, - KanbanPosition: 111, - Labels: []*models.Label{ - { - Title: "Label 3", - HexColor: trelloColorMap["blue"], + Task: models.Task{ + Title: "Test Card 5", + BucketID: 2, + KanbanPosition: 111, + Labels: []*models.Label{ + { + Title: "Label 3", + HexColor: trelloColorMap["blue"], + }, }, }, }, { - Title: "Test Card 6", - BucketID: 2, - KanbanPosition: 222, - DueDate: time1, + Task: models.Task{ + Title: "Test Card 6", + BucketID: 2, + KanbanPosition: 222, + DueDate: time1, + }, }, { - Title: "Test Card 7", - BucketID: 2, - KanbanPosition: 333, + Task: models.Task{ + Title: "Test Card 7", + BucketID: 2, + KanbanPosition: 333, + }, }, { - Title: "Test Card 8", - BucketID: 2, - KanbanPosition: 444, + Task: models.Task{ + Title: "Test Card 8", + BucketID: 2, + KanbanPosition: 444, + }, }, }, }, { - Title: "TestBoard 2", + List: models.List{ + Title: "TestBoard 2", + }, Buckets: []*models.Bucket{ { ID: 3, Title: "Test List 4", }, }, - Tasks: []*models.Task{ + Tasks: []*models.TaskWithComments{ { - Title: "Test Card 634", - BucketID: 3, - KanbanPosition: 123, + Task: models.Task{ + Title: "Test Card 634", + BucketID: 3, + KanbanPosition: 123, + }, }, }, }, { - Title: "TestBoard Archived", - IsArchived: true, + List: models.List{ + Title: "TestBoard Archived", + IsArchived: true, + }, Buckets: []*models.Bucket{ { ID: 4, Title: "Test List 5", }, }, - Tasks: []*models.Task{ + Tasks: []*models.TaskWithComments{ { - Title: "Test Card 63423", - BucketID: 4, - KanbanPosition: 123, + Task: models.Task{ + Title: "Test Card 63423", + BucketID: 4, + KanbanPosition: 123, + }, }, }, }, diff --git a/pkg/modules/migration/vikunja-file/export.zip b/pkg/modules/migration/vikunja-file/export.zip new file mode 100644 index 00000000..8163b92d Binary files /dev/null and b/pkg/modules/migration/vikunja-file/export.zip differ diff --git a/pkg/modules/migration/vikunja-file/main_test.go b/pkg/modules/migration/vikunja-file/main_test.go new file mode 100644 index 00000000..d618888d --- /dev/null +++ b/pkg/modules/migration/vikunja-file/main_test.go @@ -0,0 +1,44 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2021 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public Licensee as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public Licensee for more details. +// +// You should have received a copy of the GNU Affero General Public Licensee +// along with this program. If not, see . + +package vikunjafile + +import ( + "os" + "testing" + + "code.vikunja.io/api/pkg/events" + + "code.vikunja.io/api/pkg/config" + "code.vikunja.io/api/pkg/files" + "code.vikunja.io/api/pkg/models" + "code.vikunja.io/api/pkg/user" +) + +// TestMain is the main test function used to bootstrap the test env +func TestMain(m *testing.M) { + // Set default config + config.InitDefaultConfig() + // We need to set the root path even if we're not using the config, otherwise fixtures are not loaded correctly + config.ServiceRootpath.Set(os.Getenv("VIKUNJA_SERVICE_ROOTPATH")) + + // Some tests use the file engine, so we'll need to initialize that + files.InitTests() + user.InitTests() + models.SetupTests() + events.Fake() + os.Exit(m.Run()) +} diff --git a/pkg/modules/migration/vikunja-file/vikunja.go b/pkg/modules/migration/vikunja-file/vikunja.go new file mode 100644 index 00000000..7005744f --- /dev/null +++ b/pkg/modules/migration/vikunja-file/vikunja.go @@ -0,0 +1,204 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2021 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public Licensee as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public Licensee for more details. +// +// You should have received a copy of the GNU Affero General Public Licensee +// along with this program. If not, see . + +package vikunjafile + +import ( + "archive/zip" + "bytes" + "encoding/json" + "fmt" + "io" + "strconv" + "strings" + + "code.vikunja.io/api/pkg/db" + "code.vikunja.io/api/pkg/log" + "code.vikunja.io/api/pkg/models" + "code.vikunja.io/api/pkg/modules/migration" + "code.vikunja.io/api/pkg/user" +) + +const logPrefix = "[Vikunja File Import] " + +type FileMigrator struct { +} + +// Name is used to get the name of the vikunja-file migration - we're using the docs here to annotate the status route. +// @Summary Get migration status +// @Description Returns if the current user already did the migation or not. This is useful to show a confirmation message in the frontend if the user is trying to do the same migration again. +// @tags migration +// @Produce json +// @Security JWTKeyAuth +// @Success 200 {object} migration.Status "The migration status" +// @Failure 500 {object} models.Message "Internal server error" +// @Router /migration/vikunja-file/status [get] +func (v *FileMigrator) Name() string { + return "vikunja-file" +} + +// Migrate takes a vikunja file export, parses it and imports everything in it into Vikunja. +// @Summary Import all lists, tasks etc. from a Vikunja data export +// @Description Imports all projects, tasks, notes, reminders, subtasks and files from a Vikunjda data export into Vikunja. +// @tags migration +// @Accept json +// @Produce json +// @Security JWTKeyAuth +// @Param import formData string true "The Vikunja export zip file." +// @Success 200 {object} models.Message "A message telling you everything was migrated successfully." +// @Failure 500 {object} models.Message "Internal server error" +// @Router /migration/vikunja-file/migrate [post] +func (v *FileMigrator) Migrate(user *user.User, file io.ReaderAt, size int64) error { + r, err := zip.NewReader(file, size) + if err != nil { + return fmt.Errorf("could not open import file: %s", err) + } + + log.Debugf(logPrefix+"Importing a zip file containing %d files", len(r.File)) + + var dataFile *zip.File + var filterFile *zip.File + storedFiles := make(map[int64]*zip.File) + for _, f := range r.File { + if strings.HasPrefix(f.Name, "files/") { + fname := strings.ReplaceAll(f.Name, "files/", "") + id, err := strconv.ParseInt(fname, 10, 64) + if err != nil { + return fmt.Errorf("could not convert file id: %s", err) + } + storedFiles[id] = f + log.Debugf(logPrefix + "Found a blob file") + continue + } + if f.Name == "data.json" { + dataFile = f + log.Debugf(logPrefix + "Found a data file") + continue + } + if f.Name == "filters.json" { + filterFile = f + log.Debugf(logPrefix + "Found a filter file") + } + } + + if dataFile == nil { + return fmt.Errorf("no data file provided") + } + + log.Debugf(logPrefix + "") + + ////// + // Import the bulk of Vikunja data + df, err := dataFile.Open() + if err != nil { + return fmt.Errorf("could not open data file: %s", err) + } + defer df.Close() + + var bufData bytes.Buffer + if _, err := bufData.ReadFrom(df); err != nil { + return fmt.Errorf("could not read data file: %s", err) + } + + namespaces := []*models.NamespaceWithListsAndTasks{} + if err := json.Unmarshal(bufData.Bytes(), &namespaces); err != nil { + return fmt.Errorf("could not read data: %s", err) + } + + for _, n := range namespaces { + for _, l := range n.Lists { + if b, exists := storedFiles[l.BackgroundFileID]; exists { + bf, err := b.Open() + if err != nil { + return fmt.Errorf("could not open list background file %d for reading: %s", l.BackgroundFileID, err) + } + var buf bytes.Buffer + if _, err := buf.ReadFrom(bf); err != nil { + return fmt.Errorf("could not read list background file %d: %s", l.BackgroundFileID, err) + } + + l.BackgroundInformation = &buf + } + + for _, t := range l.Tasks { + for _, label := range t.Labels { + label.ID = 0 + } + for _, comment := range t.Comments { + comment.ID = 0 + } + for _, attachment := range t.Attachments { + af, err := storedFiles[attachment.File.ID].Open() + if err != nil { + return fmt.Errorf("could not open attachment %d for reading: %s", attachment.ID, err) + } + var buf bytes.Buffer + if _, err := buf.ReadFrom(af); err != nil { + return fmt.Errorf("could not read attachment %d: %s", attachment.ID, err) + } + + attachment.ID = 0 + attachment.File.ID = 0 + attachment.File.FileContent = buf.Bytes() + } + } + } + } + + err = migration.InsertFromStructure(namespaces, user) + if err != nil { + return fmt.Errorf("could not insert data: %s", err) + } + + if filterFile == nil { + log.Debugf(logPrefix + "No filter file found") + return nil + } + + /////// + // Import filters + ff, err := filterFile.Open() + if err != nil { + return fmt.Errorf("could not open filters file: %s", err) + } + defer ff.Close() + + var bufFilter bytes.Buffer + if _, err := bufFilter.ReadFrom(ff); err != nil { + return fmt.Errorf("could not read filters file: %s", err) + } + + filters := []*models.SavedFilter{} + if err := json.Unmarshal(bufFilter.Bytes(), &filters); err != nil { + return fmt.Errorf("could not read filter data: %s", err) + } + + log.Debugf(logPrefix+"Importing %d saved filters", len(filters)) + + s := db.NewSession() + defer s.Close() + + for _, f := range filters { + f.ID = 0 + err = f.Create(s, user) + if err != nil { + _ = s.Rollback() + return err + } + } + + return s.Commit() +} diff --git a/pkg/modules/migration/vikunja-file/vikunja_test.go b/pkg/modules/migration/vikunja-file/vikunja_test.go new file mode 100644 index 00000000..5d5361a4 --- /dev/null +++ b/pkg/modules/migration/vikunja-file/vikunja_test.go @@ -0,0 +1,79 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2021 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public Licensee as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public Licensee for more details. +// +// You should have received a copy of the GNU Affero General Public Licensee +// along with this program. If not, see . + +package vikunjafile + +import ( + "os" + "testing" + + "code.vikunja.io/api/pkg/config" + "code.vikunja.io/api/pkg/db" + "code.vikunja.io/api/pkg/user" + "github.com/stretchr/testify/assert" +) + +func TestVikunjaFileMigrator_Migrate(t *testing.T) { + db.LoadAndAssertFixtures(t) + + m := &FileMigrator{} + u := &user.User{ID: 1} + + f, err := os.Open(config.ServiceRootpath.GetString() + "/pkg/modules/migration/vikunja-file/export.zip") + if err != nil { + t.Fatalf("Could not open file: %s", err) + } + defer f.Close() + s, err := f.Stat() + if err != nil { + t.Fatalf("Could not stat file: %s", err) + } + + err = m.Migrate(u, f, s.Size()) + assert.NoError(t, err) + db.AssertExists(t, "namespaces", map[string]interface{}{ + "title": "test", + "owner_id": u.ID, + }, false) + db.AssertExists(t, "lists", map[string]interface{}{ + "title": "Test list", + "owner_id": u.ID, + }, false) + db.AssertExists(t, "lists", map[string]interface{}{ + "title": "A list with a background", + "owner_id": u.ID, + }, false) + db.AssertExists(t, "tasks", map[string]interface{}{ + "title": "Some other task", + "created_by_id": u.ID, + }, false) + db.AssertExists(t, "task_comments", map[string]interface{}{ + "comment": "This is a comment", + "author_id": u.ID, + }, false) + db.AssertExists(t, "files", map[string]interface{}{ + "name": "cristiano-mozzillo-v3d5uBB26yA-unsplash.jpg", + "created_by_id": u.ID, + }, false) + db.AssertExists(t, "labels", map[string]interface{}{ + "title": "test", + "created_by_id": u.ID, + }, false) + db.AssertExists(t, "buckets", map[string]interface{}{ + "title": "Test Bucket", + "created_by_id": u.ID, + }, false) +} diff --git a/pkg/modules/migration/wunderlist/wunderlist.go b/pkg/modules/migration/wunderlist/wunderlist.go index 0a5a4179..fea6722e 100644 --- a/pkg/modules/migration/wunderlist/wunderlist.go +++ b/pkg/modules/migration/wunderlist/wunderlist.go @@ -142,11 +142,13 @@ type wunderlistContents struct { subtasks []*subtask } -func convertListForFolder(listID int, list *list, content *wunderlistContents) (*models.List, error) { +func convertListForFolder(listID int, list *list, content *wunderlistContents) (*models.ListWithTasksAndBuckets, error) { - l := &models.List{ - Title: list.Title, - Created: list.CreatedAt, + l := &models.ListWithTasksAndBuckets{ + List: models.List{ + Title: list.Title, + Created: list.CreatedAt, + }, } // Find all tasks belonging to this list and put them in @@ -233,13 +235,13 @@ func convertListForFolder(listID int, list *list, content *wunderlistContents) ( } } - l.Tasks = append(l.Tasks, newTask) + l.Tasks = append(l.Tasks, &models.TaskWithComments{Task: *newTask}) } } return l, nil } -func convertWunderlistToVikunja(content *wunderlistContents) (fullVikunjaHierachie []*models.NamespaceWithLists, err error) { +func convertWunderlistToVikunja(content *wunderlistContents) (fullVikunjaHierachie []*models.NamespaceWithListsAndTasks, err error) { // Make a map from the list with the key being list id for easier handling listMap := make(map[int]*list, len(content.lists)) @@ -249,7 +251,7 @@ func convertWunderlistToVikunja(content *wunderlistContents) (fullVikunjaHierach // First, we look through all folders and create namespaces for them. for _, folder := range content.folders { - namespace := &models.NamespaceWithLists{ + namespace := &models.NamespaceWithListsAndTasks{ Namespace: models.Namespace{ Title: folder.Title, Created: folder.CreatedAt, @@ -276,7 +278,7 @@ func convertWunderlistToVikunja(content *wunderlistContents) (fullVikunjaHierach // At the end, loop over all lists which don't belong to a namespace and put them in a default namespace if len(listMap) > 0 { - newNamespace := &models.NamespaceWithLists{ + newNamespace := &models.NamespaceWithListsAndTasks{ Namespace: models.Namespace{ Title: "Migrated from wunderlist", }, diff --git a/pkg/modules/migration/wunderlist/wunderlist_test.go b/pkg/modules/migration/wunderlist/wunderlist_test.go index 8f6ef224..3385ac79 100644 --- a/pkg/modules/migration/wunderlist/wunderlist_test.go +++ b/pkg/modules/migration/wunderlist/wunderlist_test.go @@ -194,49 +194,55 @@ func TestWunderlistParsing(t *testing.T) { }, } - expectedHierachie := []*models.NamespaceWithLists{ + expectedHierachie := []*models.NamespaceWithListsAndTasks{ { Namespace: models.Namespace{ Title: "Lorem Ipsum", Created: time1, Updated: time2, }, - Lists: []*models.List{ + Lists: []*models.ListWithTasksAndBuckets{ { - Created: time1, - Title: "Lorem1", - Tasks: []*models.Task{ + List: models.List{ + Created: time1, + Title: "Lorem1", + }, + Tasks: []*models.TaskWithComments{ { - Title: "Ipsum1", - DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()), - Created: time1, - Description: "Lorem Ipsum dolor sit amet", - Attachments: []*models.TaskAttachment{ - { - File: &files.File{ - Name: "file.md", - Mime: "text/plain", - Size: 12345, - Created: time2, - FileContent: exampleFile, + Task: models.Task{ + Title: "Ipsum1", + DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()), + Created: time1, + Description: "Lorem Ipsum dolor sit amet", + Attachments: []*models.TaskAttachment{ + { + File: &files.File{ + Name: "file.md", + Mime: "text/plain", + Size: 12345, + Created: time2, + FileContent: exampleFile, + }, + Created: time2, }, - Created: time2, }, + Reminders: []time.Time{time4}, }, - Reminders: []time.Time{time4}, }, { - Title: "Ipsum2", - DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()), - Created: time1, - Description: "Lorem Ipsum dolor sit amet", - RelatedTasks: map[models.RelationKind][]*models.Task{ - models.RelationKindSubtask: { - { - Title: "LoremSub1", - }, - { - Title: "LoremSub2", + Task: models.Task{ + Title: "Ipsum2", + DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()), + Created: time1, + Description: "Lorem Ipsum dolor sit amet", + RelatedTasks: map[models.RelationKind][]*models.Task{ + models.RelationKindSubtask: { + { + Title: "LoremSub1", + }, + { + Title: "LoremSub2", + }, }, }, }, @@ -244,38 +250,44 @@ func TestWunderlistParsing(t *testing.T) { }, }, { - Created: time1, - Title: "Lorem2", - Tasks: []*models.Task{ + List: models.List{ + Created: time1, + Title: "Lorem2", + }, + Tasks: []*models.TaskWithComments{ { - Title: "Ipsum3", - Done: true, - DoneAt: time1, - DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()), - Created: time1, - Description: "Lorem Ipsum dolor sit amet", - Attachments: []*models.TaskAttachment{ - { - File: &files.File{ - Name: "file2.md", - Mime: "text/plain", - Size: 12345, - Created: time3, - FileContent: exampleFile, - }, - Created: time3, - }, - }, - }, - { - Title: "Ipsum4", - DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()), - Created: time1, - Reminders: []time.Time{time3}, - RelatedTasks: map[models.RelationKind][]*models.Task{ - models.RelationKindSubtask: { + Task: models.Task{ + Title: "Ipsum3", + Done: true, + DoneAt: time1, + DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()), + Created: time1, + Description: "Lorem Ipsum dolor sit amet", + Attachments: []*models.TaskAttachment{ { - Title: "LoremSub3", + File: &files.File{ + Name: "file2.md", + Mime: "text/plain", + Size: 12345, + Created: time3, + FileContent: exampleFile, + }, + Created: time3, + }, + }, + }, + }, + { + Task: models.Task{ + Title: "Ipsum4", + DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()), + Created: time1, + Reminders: []time.Time{time3}, + RelatedTasks: map[models.RelationKind][]*models.Task{ + models.RelationKindSubtask: { + { + Title: "LoremSub3", + }, }, }, }, @@ -283,52 +295,68 @@ func TestWunderlistParsing(t *testing.T) { }, }, { - Created: time1, - Title: "Lorem3", - Tasks: []*models.Task{ + List: models.List{ + Created: time1, + Title: "Lorem3", + }, + Tasks: []*models.TaskWithComments{ { - Title: "Ipsum5", - DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()), - Created: time1, + Task: models.Task{ + Title: "Ipsum5", + DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()), + Created: time1, + }, }, { - Title: "Ipsum6", - DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()), - Created: time1, - Done: true, - DoneAt: time1, + Task: models.Task{ + Title: "Ipsum6", + DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()), + Created: time1, + Done: true, + DoneAt: time1, + }, }, { - Title: "Ipsum7", - DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()), - Created: time1, - Done: true, - DoneAt: time1, + Task: models.Task{ + Title: "Ipsum7", + DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()), + Created: time1, + Done: true, + DoneAt: time1, + }, }, { - Title: "Ipsum8", - DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()), - Created: time1, + Task: models.Task{ + Title: "Ipsum8", + DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()), + Created: time1, + }, }, }, }, { - Created: time1, - Title: "Lorem4", - Tasks: []*models.Task{ + List: models.List{ + Created: time1, + Title: "Lorem4", + }, + Tasks: []*models.TaskWithComments{ { - Title: "Ipsum9", - DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()), - Created: time1, - Done: true, - DoneAt: time1, + Task: models.Task{ + Title: "Ipsum9", + DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()), + Created: time1, + Done: true, + DoneAt: time1, + }, }, { - Title: "Ipsum10", - DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()), - Created: time1, - Done: true, - DoneAt: time1, + Task: models.Task{ + Title: "Ipsum10", + DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()), + Created: time1, + Done: true, + DoneAt: time1, + }, }, }, }, @@ -338,10 +366,12 @@ func TestWunderlistParsing(t *testing.T) { Namespace: models.Namespace{ Title: "Migrated from wunderlist", }, - Lists: []*models.List{ + Lists: []*models.ListWithTasksAndBuckets{ { - Created: time4, - Title: "List without a namespace", + List: models.List{ + Created: time4, + Title: "List without a namespace", + }, }, }, }, diff --git a/pkg/routes/api/v1/info.go b/pkg/routes/api/v1/info.go index 2ae12c10..74afd75b 100644 --- a/pkg/routes/api/v1/info.go +++ b/pkg/routes/api/v1/info.go @@ -19,6 +19,8 @@ package v1 import ( "net/http" + vikunja_file "code.vikunja.io/api/pkg/modules/migration/vikunja-file" + microsofttodo "code.vikunja.io/api/pkg/modules/migration/microsoft-todo" "code.vikunja.io/api/pkg/modules/migration/trello" @@ -91,6 +93,9 @@ func Info(c echo.Context) error { CaldavEnabled: config.ServiceEnableCaldav.GetBool(), EmailRemindersEnabled: config.ServiceEnableEmailReminders.GetBool(), UserDeletionEnabled: config.ServiceEnableUserDeletion.GetBool(), + AvailableMigrators: []string{ + (&vikunja_file.FileMigrator{}).Name(), + }, Legal: legalInfo{ ImprintURL: config.LegalImprintURL.GetString(), PrivacyPolicyURL: config.LegalPrivacyURL.GetString(), diff --git a/pkg/routes/api/v1/user_deletion.go b/pkg/routes/api/v1/user_deletion.go index 391c761e..ac1d18c8 100644 --- a/pkg/routes/api/v1/user_deletion.go +++ b/pkg/routes/api/v1/user_deletion.go @@ -27,7 +27,7 @@ import ( "github.com/labstack/echo/v4" ) -type UserDeletionRequest struct { +type UserPasswordConfirmation struct { Password string `json:"password" valid:"required"` } @@ -41,13 +41,13 @@ type UserDeletionRequestConfirm struct { // @tags user // @Accept json // @Produce json -// @Param credentials body v1.UserDeletionRequest true "The user password." +// @Param credentials body v1.UserPasswordConfirmation true "The user password." // @Success 200 {object} models.Message // @Failure 412 {object} web.HTTPError "Bad password provided." // @Failure 500 {object} models.Message "Internal error" // @Router /user/deletion/request [post] func UserRequestDeletion(c echo.Context) error { - var deletionRequest UserDeletionRequest + var deletionRequest UserPasswordConfirmation if err := c.Bind(&deletionRequest); err != nil { return echo.NewHTTPError(http.StatusBadRequest, "No password provided.") } @@ -149,13 +149,13 @@ func UserConfirmDeletion(c echo.Context) error { // @tags user // @Accept json // @Produce json -// @Param credentials body v1.UserDeletionRequest true "The user password to confirm." +// @Param credentials body v1.UserPasswordConfirmation true "The user password to confirm." // @Success 200 {object} models.Message // @Failure 412 {object} web.HTTPError "Bad password provided." // @Failure 500 {object} models.Message "Internal error" // @Router /user/deletion/cancel [post] func UserCancelDeletion(c echo.Context) error { - var deletionRequest UserDeletionRequest + var deletionRequest UserPasswordConfirmation if err := c.Bind(&deletionRequest); err != nil { return echo.NewHTTPError(http.StatusBadRequest, "No password provided.") } diff --git a/pkg/routes/api/v1/user_export.go b/pkg/routes/api/v1/user_export.go new file mode 100644 index 00000000..c731d0cf --- /dev/null +++ b/pkg/routes/api/v1/user_export.go @@ -0,0 +1,136 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2021 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public Licensee as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public Licensee for more details. +// +// You should have received a copy of the GNU Affero General Public Licensee +// along with this program. If not, see . + +package v1 + +import ( + "net/http" + + "code.vikunja.io/api/pkg/db" + "code.vikunja.io/api/pkg/events" + "code.vikunja.io/api/pkg/files" + "code.vikunja.io/api/pkg/models" + "code.vikunja.io/api/pkg/user" + "code.vikunja.io/web/handler" + "github.com/labstack/echo/v4" + "xorm.io/xorm" +) + +func checkExportRequest(c echo.Context) (s *xorm.Session, u *user.User, err error) { + var pass UserPasswordConfirmation + if err := c.Bind(&pass); err != nil { + return nil, nil, echo.NewHTTPError(http.StatusBadRequest, "No password provided.") + } + + err = c.Validate(pass) + if err != nil { + return nil, nil, echo.NewHTTPError(http.StatusBadRequest, err) + } + + s = db.NewSession() + defer s.Close() + + err = s.Begin() + if err != nil { + return nil, nil, handler.HandleHTTPError(err, c) + } + + u, err = user.GetCurrentUserFromDB(s, c) + if err != nil { + _ = s.Rollback() + return nil, nil, handler.HandleHTTPError(err, c) + } + + err = user.CheckUserPassword(u, pass.Password) + if err != nil { + _ = s.Rollback() + return nil, nil, handler.HandleHTTPError(err, c) + } + + return +} + +// RequestUserDataExport is the handler to request a user data export +// @Summary Request a user data export. +// @tags user +// @Accept json +// @Produce json +// @Security JWTKeyAuth +// @Param password body v1.UserPasswordConfirmation true "User password to confirm the data export request." +// @Success 200 {object} models.Message +// @Failure 400 {object} web.HTTPError "Something's invalid." +// @Failure 500 {object} models.Message "Internal server error." +// @Router /user/export/request [post] +func RequestUserDataExport(c echo.Context) error { + s, u, err := checkExportRequest(c) + if err != nil { + return err + } + + err = events.Dispatch(&models.UserDataExportRequestedEvent{ + User: u, + }) + if err != nil { + _ = s.Rollback() + return handler.HandleHTTPError(err, c) + } + + err = s.Commit() + if err != nil { + _ = s.Rollback() + return handler.HandleHTTPError(err, c) + } + + return c.JSON(http.StatusOK, models.Message{Message: "Successfully requested data export. We will send you an email when it's ready."}) +} + +// DownloadUserDataExport is the handler to download a created user data export +// @Summary Download a user data export. +// @tags user +// @Accept json +// @Produce json +// @Security JWTKeyAuth +// @Param password body v1.UserPasswordConfirmation true "User password to confirm the download." +// @Success 200 {object} models.Message +// @Failure 400 {object} web.HTTPError "Something's invalid." +// @Failure 500 {object} models.Message "Internal server error." +// @Router /user/export/download [post] +func DownloadUserDataExport(c echo.Context) error { + s, u, err := checkExportRequest(c) + if err != nil { + return err + } + + err = s.Commit() + if err != nil { + _ = s.Rollback() + return handler.HandleHTTPError(err, c) + } + + // Download + exportFile := &files.File{ID: u.ExportFileID} + err = exportFile.LoadFileMetaByID() + if err != nil { + return handler.HandleHTTPError(err, c) + } + err = exportFile.LoadFileByID() + if err != nil { + return handler.HandleHTTPError(err, c) + } + + http.ServeContent(c.Response(), c.Request(), exportFile.Name, exportFile.Created, exportFile.File) + return nil +} diff --git a/pkg/routes/caldav/handler.go b/pkg/routes/caldav/handler.go index 8d3c4bb7..8d748130 100644 --- a/pkg/routes/caldav/handler.go +++ b/pkg/routes/caldav/handler.go @@ -56,7 +56,7 @@ func ListHandler(c echo.Context) error { } storage := &VikunjaCaldavListStorage{ - list: &models.List{ID: listID}, + list: &models.ListWithTasksAndBuckets{List: models.List{ID: listID}}, user: u, } @@ -102,7 +102,7 @@ func TaskHandler(c echo.Context) error { taskUID := strings.TrimSuffix(c.Param("task"), ".ics") storage := &VikunjaCaldavListStorage{ - list: &models.List{ID: listID}, + list: &models.ListWithTasksAndBuckets{List: models.List{ID: listID}}, task: &models.Task{UID: taskUID}, user: u, } diff --git a/pkg/routes/caldav/listStorageProvider.go b/pkg/routes/caldav/listStorageProvider.go index 9abe0341..0d1692c1 100644 --- a/pkg/routes/caldav/listStorageProvider.go +++ b/pkg/routes/caldav/listStorageProvider.go @@ -39,7 +39,7 @@ const ListBasePath = DavBasePath + `lists` // VikunjaCaldavListStorage represents a list storage type VikunjaCaldavListStorage struct { // Used when handling a list - list *models.List + list *models.ListWithTasksAndBuckets // Used when handling a single task, like updating task *models.Task // The current user @@ -109,7 +109,9 @@ func (vcls *VikunjaCaldavListStorage) GetResources(rpath string, withChildren bo var resources []data.Resource for _, l := range lists { rr := VikunjaListResourceAdapter{ - list: l, + list: &models.ListWithTasksAndBuckets{ + List: *l, + }, isCollection: true, } r := data.NewResource(ListBasePath+"/"+strconv.FormatInt(l.ID, 10), &rr) @@ -172,10 +174,10 @@ func (vcls *VikunjaCaldavListStorage) GetResourcesByFilters(rpath string, filter for _, t := range vcls.list.Tasks { rr := VikunjaListResourceAdapter{ list: vcls.list, - task: t, + task: &t.Task, isCollection: false, } - r := data.NewResource(getTaskURL(t), &rr) + r := data.NewResource(getTaskURL(&t.Task), &rr) r.Name = t.Title resources = append(resources, r) } @@ -368,8 +370,8 @@ func (vcls *VikunjaCaldavListStorage) DeleteResource(rpath string) error { // VikunjaListResourceAdapter holds the actual resource type VikunjaListResourceAdapter struct { - list *models.List - listTasks []*models.Task + list *models.ListWithTasksAndBuckets + listTasks []*models.TaskWithComments task *models.Task isPrincipal bool @@ -415,7 +417,7 @@ func (vlra *VikunjaListResourceAdapter) GetContent() string { } if vlra.task != nil { - list := models.List{Tasks: []*models.Task{vlra.task}} + list := models.ListWithTasksAndBuckets{Tasks: []*models.TaskWithComments{{Task: *vlra.task}}} return caldav.GetCaldavTodosForTasks(&list, list.Tasks) } @@ -479,8 +481,10 @@ func (vcls *VikunjaCaldavListStorage) getListRessource(isCollection bool) (rr Vi panic("Tasks returned from TaskCollection.ReadAll are not []*models.Task!") } - listTasks = tasks - vcls.list.Tasks = tasks + for _, t := range tasks { + listTasks = append(listTasks, &models.TaskWithComments{Task: *t}) + } + vcls.list.Tasks = listTasks } if err := s.Commit(); err != nil { diff --git a/pkg/routes/routes.go b/pkg/routes/routes.go index 678cdfe7..b340aa6d 100644 --- a/pkg/routes/routes.go +++ b/pkg/routes/routes.go @@ -52,6 +52,8 @@ import ( "strings" "time" + vikunja_file "code.vikunja.io/api/pkg/modules/migration/vikunja-file" + "code.vikunja.io/api/pkg/config" "code.vikunja.io/api/pkg/db" "code.vikunja.io/api/pkg/log" @@ -303,6 +305,8 @@ func registerAPIRoutes(a *echo.Group) { u.POST("/settings/avatar", apiv1.ChangeUserAvatarProvider) u.PUT("/settings/avatar/upload", apiv1.UploadAvatar) u.POST("/settings/general", apiv1.UpdateGeneralUserSettings) + u.POST("/export/request", apiv1.RequestUserDataExport) + u.POST("/export/download", apiv1.DownloadUserDataExport) if config.ServiceEnableTotp.GetBool() { u.GET("/settings/totp", apiv1.UserTOTP) @@ -563,7 +567,35 @@ func registerAPIRoutes(a *echo.Group) { // Migrations m := a.Group("/migration") + registerMigrations(m) + // List Backgrounds + if config.BackgroundsEnabled.GetBool() { + a.GET("/lists/:list/background", backgroundHandler.GetListBackground) + a.DELETE("/lists/:list/background", backgroundHandler.RemoveListBackground) + if config.BackgroundsUploadEnabled.GetBool() { + uploadBackgroundProvider := &backgroundHandler.BackgroundProvider{ + Provider: func() background.Provider { + return &upload.Provider{} + }, + } + a.PUT("/lists/:list/backgrounds/upload", uploadBackgroundProvider.UploadBackground) + } + if config.BackgroundsUnsplashEnabled.GetBool() { + unsplashBackgroundProvider := &backgroundHandler.BackgroundProvider{ + Provider: func() background.Provider { + return &unsplash.Provider{} + }, + } + a.GET("/backgrounds/unsplash/search", unsplashBackgroundProvider.SearchBackgrounds) + a.POST("/lists/:list/backgrounds/unsplash", unsplashBackgroundProvider.SetBackground) + a.GET("/backgrounds/unsplash/images/:image/thumb", unsplash.ProxyUnsplashThumb) + a.GET("/backgrounds/unsplash/images/:image", unsplash.ProxyUnsplashImage) + } + } +} + +func registerMigrations(m *echo.Group) { // Wunderlist if config.MigrationWunderlistEnable.GetBool() { wunderlistMigrationHandler := &migrationHandler.MigrationWeb{ @@ -604,30 +636,12 @@ func registerAPIRoutes(a *echo.Group) { microsoftTodoMigrationHandler.RegisterRoutes(m) } - // List Backgrounds - if config.BackgroundsEnabled.GetBool() { - a.GET("/lists/:list/background", backgroundHandler.GetListBackground) - a.DELETE("/lists/:list/background", backgroundHandler.RemoveListBackground) - if config.BackgroundsUploadEnabled.GetBool() { - uploadBackgroundProvider := &backgroundHandler.BackgroundProvider{ - Provider: func() background.Provider { - return &upload.Provider{} - }, - } - a.PUT("/lists/:list/backgrounds/upload", uploadBackgroundProvider.UploadBackground) - } - if config.BackgroundsUnsplashEnabled.GetBool() { - unsplashBackgroundProvider := &backgroundHandler.BackgroundProvider{ - Provider: func() background.Provider { - return &unsplash.Provider{} - }, - } - a.GET("/backgrounds/unsplash/search", unsplashBackgroundProvider.SearchBackgrounds) - a.POST("/lists/:list/backgrounds/unsplash", unsplashBackgroundProvider.SetBackground) - a.GET("/backgrounds/unsplash/images/:image/thumb", unsplash.ProxyUnsplashThumb) - a.GET("/backgrounds/unsplash/images/:image", unsplash.ProxyUnsplashImage) - } + vikunjaFileMigrationHandler := &migrationHandler.FileMigratorWeb{ + MigrationStruct: func() migration.FileMigrator { + return &vikunja_file.FileMigrator{} + }, } + vikunjaFileMigrationHandler.RegisterRoutes(m) } func registerCalDavRoutes(c *echo.Group) { diff --git a/pkg/swagger/docs.go b/pkg/swagger/docs.go index 0815bd4e..fc3f9079 100644 --- a/pkg/swagger/docs.go +++ b/pkg/swagger/docs.go @@ -2921,6 +2921,80 @@ var doc = `{ } } }, + "/migration/vikunja-file/migrate": { + "post": { + "security": [ + { + "JWTKeyAuth": [] + } + ], + "description": "Imports all projects, tasks, notes, reminders, subtasks and files from a Vikunjda data export into Vikunja.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "migration" + ], + "summary": "Import all lists, tasks etc. from a Vikunja data export", + "parameters": [ + { + "type": "string", + "description": "The Vikunja export zip file.", + "name": "import", + "in": "formData", + "required": true + } + ], + "responses": { + "200": { + "description": "A message telling you everything was migrated successfully.", + "schema": { + "$ref": "#/definitions/models.Message" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/models.Message" + } + } + } + } + }, + "/migration/vikunja-file/status": { + "get": { + "security": [ + { + "JWTKeyAuth": [] + } + ], + "description": "Returns if the current user already did the migation or not. This is useful to show a confirmation message in the frontend if the user is trying to do the same migration again.", + "produces": [ + "application/json" + ], + "tags": [ + "migration" + ], + "summary": "Get migration status", + "responses": { + "200": { + "description": "The migration status", + "schema": { + "$ref": "#/definitions/migration.Status" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/models.Message" + } + } + } + } + }, "/migration/wunderlist/auth": { "get": { "security": [ @@ -6335,7 +6409,7 @@ var doc = `{ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/v1.UserDeletionRequest" + "$ref": "#/definitions/v1.UserPasswordConfirmation" } } ], @@ -6427,7 +6501,7 @@ var doc = `{ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/v1.UserDeletionRequest" + "$ref": "#/definitions/v1.UserPasswordConfirmation" } } ], @@ -6453,6 +6527,106 @@ var doc = `{ } } }, + "/user/export/download": { + "post": { + "security": [ + { + "JWTKeyAuth": [] + } + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "user" + ], + "summary": "Download a user data export.", + "parameters": [ + { + "description": "User password to confirm the download.", + "name": "password", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1.UserPasswordConfirmation" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/models.Message" + } + }, + "400": { + "description": "Something's invalid.", + "schema": { + "$ref": "#/definitions/web.HTTPError" + } + }, + "500": { + "description": "Internal server error.", + "schema": { + "$ref": "#/definitions/models.Message" + } + } + } + } + }, + "/user/export/request": { + "post": { + "security": [ + { + "JWTKeyAuth": [] + } + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "user" + ], + "summary": "Request a user data export.", + "parameters": [ + { + "description": "User password to confirm the data export request.", + "name": "password", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1.UserPasswordConfirmation" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/models.Message" + } + }, + "400": { + "description": "Something's invalid.", + "schema": { + "$ref": "#/definitions/web.HTTPError" + } + }, + "500": { + "description": "Internal server error.", + "schema": { + "$ref": "#/definitions/models.Message" + } + } + } + } + }, "/user/password": { "post": { "security": [ @@ -8694,14 +8868,6 @@ var doc = `{ } } }, - "v1.UserDeletionRequest": { - "type": "object", - "properties": { - "password": { - "type": "string" - } - } - }, "v1.UserDeletionRequestConfirm": { "type": "object", "properties": { @@ -8721,6 +8887,14 @@ var doc = `{ } } }, + "v1.UserPasswordConfirmation": { + "type": "object", + "properties": { + "password": { + "type": "string" + } + } + }, "v1.UserSettings": { "type": "object", "properties": { diff --git a/pkg/swagger/swagger.json b/pkg/swagger/swagger.json index 0d4bc75e..7c456a91 100644 --- a/pkg/swagger/swagger.json +++ b/pkg/swagger/swagger.json @@ -2904,6 +2904,80 @@ } } }, + "/migration/vikunja-file/migrate": { + "post": { + "security": [ + { + "JWTKeyAuth": [] + } + ], + "description": "Imports all projects, tasks, notes, reminders, subtasks and files from a Vikunjda data export into Vikunja.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "migration" + ], + "summary": "Import all lists, tasks etc. from a Vikunja data export", + "parameters": [ + { + "type": "string", + "description": "The Vikunja export zip file.", + "name": "import", + "in": "formData", + "required": true + } + ], + "responses": { + "200": { + "description": "A message telling you everything was migrated successfully.", + "schema": { + "$ref": "#/definitions/models.Message" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/models.Message" + } + } + } + } + }, + "/migration/vikunja-file/status": { + "get": { + "security": [ + { + "JWTKeyAuth": [] + } + ], + "description": "Returns if the current user already did the migation or not. This is useful to show a confirmation message in the frontend if the user is trying to do the same migration again.", + "produces": [ + "application/json" + ], + "tags": [ + "migration" + ], + "summary": "Get migration status", + "responses": { + "200": { + "description": "The migration status", + "schema": { + "$ref": "#/definitions/migration.Status" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/models.Message" + } + } + } + } + }, "/migration/wunderlist/auth": { "get": { "security": [ @@ -6318,7 +6392,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/v1.UserDeletionRequest" + "$ref": "#/definitions/v1.UserPasswordConfirmation" } } ], @@ -6410,7 +6484,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/v1.UserDeletionRequest" + "$ref": "#/definitions/v1.UserPasswordConfirmation" } } ], @@ -6436,6 +6510,106 @@ } } }, + "/user/export/download": { + "post": { + "security": [ + { + "JWTKeyAuth": [] + } + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "user" + ], + "summary": "Download a user data export.", + "parameters": [ + { + "description": "User password to confirm the download.", + "name": "password", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1.UserPasswordConfirmation" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/models.Message" + } + }, + "400": { + "description": "Something's invalid.", + "schema": { + "$ref": "#/definitions/web.HTTPError" + } + }, + "500": { + "description": "Internal server error.", + "schema": { + "$ref": "#/definitions/models.Message" + } + } + } + } + }, + "/user/export/request": { + "post": { + "security": [ + { + "JWTKeyAuth": [] + } + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "user" + ], + "summary": "Request a user data export.", + "parameters": [ + { + "description": "User password to confirm the data export request.", + "name": "password", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1.UserPasswordConfirmation" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/models.Message" + } + }, + "400": { + "description": "Something's invalid.", + "schema": { + "$ref": "#/definitions/web.HTTPError" + } + }, + "500": { + "description": "Internal server error.", + "schema": { + "$ref": "#/definitions/models.Message" + } + } + } + } + }, "/user/password": { "post": { "security": [ @@ -8677,14 +8851,6 @@ } } }, - "v1.UserDeletionRequest": { - "type": "object", - "properties": { - "password": { - "type": "string" - } - } - }, "v1.UserDeletionRequestConfirm": { "type": "object", "properties": { @@ -8704,6 +8870,14 @@ } } }, + "v1.UserPasswordConfirmation": { + "type": "object", + "properties": { + "password": { + "type": "string" + } + } + }, "v1.UserSettings": { "type": "object", "properties": { diff --git a/pkg/swagger/swagger.yaml b/pkg/swagger/swagger.yaml index a9eb87d1..edc00270 100644 --- a/pkg/swagger/swagger.yaml +++ b/pkg/swagger/swagger.yaml @@ -1185,11 +1185,6 @@ definitions: email), `upload`, `initials`, `default`. type: string type: object - v1.UserDeletionRequest: - properties: - password: - type: string - type: object v1.UserDeletionRequestConfirm: properties: token: @@ -1202,6 +1197,11 @@ definitions: old_password: type: string type: object + v1.UserPasswordConfirmation: + properties: + password: + type: string + type: object v1.UserSettings: properties: default_list_id: @@ -3287,6 +3287,55 @@ paths: summary: Get migration status tags: - migration + /migration/vikunja-file/migrate: + post: + consumes: + - application/json + description: Imports all projects, tasks, notes, reminders, subtasks and files + from a Vikunjda data export into Vikunja. + parameters: + - description: The Vikunja export zip file. + in: formData + name: import + required: true + type: string + produces: + - application/json + responses: + "200": + description: A message telling you everything was migrated successfully. + schema: + $ref: '#/definitions/models.Message' + "500": + description: Internal server error + schema: + $ref: '#/definitions/models.Message' + security: + - JWTKeyAuth: [] + summary: Import all lists, tasks etc. from a Vikunja data export + tags: + - migration + /migration/vikunja-file/status: + get: + description: Returns if the current user already did the migation or not. This + is useful to show a confirmation message in the frontend if the user is trying + to do the same migration again. + produces: + - application/json + responses: + "200": + description: The migration status + schema: + $ref: '#/definitions/migration.Status' + "500": + description: Internal server error + schema: + $ref: '#/definitions/models.Message' + security: + - JWTKeyAuth: [] + summary: Get migration status + tags: + - migration /migration/wunderlist/auth: get: description: Returns the auth url where the user needs to get its auth code. @@ -5554,7 +5603,7 @@ paths: name: credentials required: true schema: - $ref: '#/definitions/v1.UserDeletionRequest' + $ref: '#/definitions/v1.UserPasswordConfirmation' produces: - application/json responses: @@ -5615,7 +5664,7 @@ paths: name: credentials required: true schema: - $ref: '#/definitions/v1.UserDeletionRequest' + $ref: '#/definitions/v1.UserPasswordConfirmation' produces: - application/json responses: @@ -5634,6 +5683,68 @@ paths: summary: Request the deletion of the user tags: - user + /user/export/download: + post: + consumes: + - application/json + parameters: + - description: User password to confirm the download. + in: body + name: password + required: true + schema: + $ref: '#/definitions/v1.UserPasswordConfirmation' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/models.Message' + "400": + description: Something's invalid. + schema: + $ref: '#/definitions/web.HTTPError' + "500": + description: Internal server error. + schema: + $ref: '#/definitions/models.Message' + security: + - JWTKeyAuth: [] + summary: Download a user data export. + tags: + - user + /user/export/request: + post: + consumes: + - application/json + parameters: + - description: User password to confirm the data export request. + in: body + name: password + required: true + schema: + $ref: '#/definitions/v1.UserPasswordConfirmation' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/models.Message' + "400": + description: Something's invalid. + schema: + $ref: '#/definitions/web.HTTPError' + "500": + description: Internal server error. + schema: + $ref: '#/definitions/models.Message' + security: + - JWTKeyAuth: [] + summary: Request a user data export. + tags: + - user /user/password: post: consumes: diff --git a/pkg/user/events.go b/pkg/user/events.go index 0b07c9ea..32338bcf 100644 --- a/pkg/user/events.go +++ b/pkg/user/events.go @@ -21,7 +21,7 @@ type CreatedEvent struct { User *User } -// TopicName defines the name for CreatedEvent +// Name defines the name for CreatedEvent func (t *CreatedEvent) Name() string { return "user.created" } diff --git a/pkg/user/user.go b/pkg/user/user.go index cec27b5a..ac1786d9 100644 --- a/pkg/user/user.go +++ b/pkg/user/user.go @@ -98,6 +98,8 @@ type User struct { DeletionScheduledAt time.Time `xorm:"datetime null" json:"-"` DeletionLastReminderSent time.Time `xorm:"datetime null" json:"-"` + ExportFileID int64 `xorm:"bigint null" json:"-"` + // A timestamp when this task was created. You cannot change this value. Created time.Time `xorm:"created not null" json:"created"` // A timestamp when this task was last updated. You cannot change this value. diff --git a/pkg/utils/write_to_zip.go b/pkg/utils/write_to_zip.go new file mode 100644 index 00000000..d6771d0e --- /dev/null +++ b/pkg/utils/write_to_zip.go @@ -0,0 +1,63 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2021 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public Licensee as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public Licensee for more details. +// +// You should have received a copy of the GNU Affero General Public Licensee +// along with this program. If not, see . + +package utils + +import ( + "archive/zip" + "fmt" + "io" + "strconv" +) + +// Change to deflate to gain better compression +// see http://golang.org/pkg/archive/zip/#pkg-constants +const CompressionUsed = zip.Deflate + +func WriteBytesToZip(filename string, data []byte, writer *zip.Writer) (err error) { + header := &zip.FileHeader{ + Name: filename, + Method: CompressionUsed, + } + w, err := writer.CreateHeader(header) + if err != nil { + return err + } + _, err = w.Write(data) + return +} + +// WriteFilesToZip writes a bunch of files from the db to a zip file. It exprects a map with the file id +// as key and its content as io.ReadCloser. +func WriteFilesToZip(files map[int64]io.ReadCloser, wr *zip.Writer) (err error) { + for fid, file := range files { + header := &zip.FileHeader{ + Name: "files/" + strconv.FormatInt(fid, 10), + Method: CompressionUsed, + } + w, err := wr.CreateHeader(header) + if err != nil { + return err + } + _, err = io.Copy(w, file) + if err != nil { + return fmt.Errorf("error writing file %d: %s", fid, err) + } + _ = file.Close() + } + + return nil +}