vikunja-api/pkg/models/export.go

367 lines
8.3 KiB
Go
Raw Normal View History

// Vikunja is a to-do list application to facilitate your life.
// Copyright 2018-2021 Vikunja and contributors. All rights reserved.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public Licensee as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public Licensee for more details.
//
// You should have received a copy of the GNU Affero General Public Licensee
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package models
import (
"archive/zip"
"encoding/json"
"fmt"
"io"
"os"
"strconv"
"time"
"code.vikunja.io/api/pkg/config"
"code.vikunja.io/api/pkg/cron"
"code.vikunja.io/api/pkg/db"
"code.vikunja.io/api/pkg/files"
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/notifications"
"code.vikunja.io/api/pkg/user"
"code.vikunja.io/api/pkg/utils"
"code.vikunja.io/api/pkg/version"
"xorm.io/xorm"
)
func ExportUserData(s *xorm.Session, u *user.User) (err error) {
exportDir := config.FilesBasePath.GetString() + "/user-export-tmp/"
err = os.MkdirAll(exportDir, 0700)
if err != nil {
return err
}
tmpFilename := exportDir + strconv.FormatInt(u.ID, 10) + "_" + time.Now().Format("2006-01-02_15-03-05") + ".zip"
// Open zip
dumpFile, err := os.Create(tmpFilename)
if err != nil {
2022-03-27 16:55:37 +02:00
return fmt.Errorf("error opening dump file: %w", err)
}
defer dumpFile.Close()
dumpWriter := zip.NewWriter(dumpFile)
defer dumpWriter.Close()
// Get the data
err = exportListsAndTasks(s, u, dumpWriter)
if err != nil {
return err
}
// Task attachment files
err = exportTaskAttachments(s, u, dumpWriter)
if err != nil {
return err
}
// Saved filters
err = exportSavedFilters(s, u, dumpWriter)
if err != nil {
return err
}
// Background files
err = exportListBackgrounds(s, u, dumpWriter)
if err != nil {
return err
}
// Vikunja Version
err = utils.WriteBytesToZip("VERSION", []byte(version.Version), dumpWriter)
if err != nil {
return err
}
// If we reuse the same file again, saving it as a file in Vikunja will save it as a file with 0 bytes in size.
// Closing and reopening does work.
dumpWriter.Close()
dumpFile.Close()
exported, err := os.Open(tmpFilename)
if err != nil {
return err
}
stat, err := exported.Stat()
if err != nil {
return err
}
exportFile, err := files.CreateWithMimeAndSession(s, exported, tmpFilename, uint64(stat.Size()), u, "application/zip")
if err != nil {
return err
}
// Save the file id with the user
u.ExportFileID = exportFile.ID
_, err = s.Cols("export_file_id").Update(u)
if err != nil {
return
}
// Remove the old file
err = os.Remove(exported.Name())
if err != nil {
return err
}
// Send a notification
return notifications.Notify(u, &DataExportReadyNotification{
User: u,
})
}
func exportListsAndTasks(s *xorm.Session, u *user.User, wr *zip.Writer) (err error) {
namspaces, _, _, err := (&Namespace{IsArchived: true}).ReadAll(s, u, "", -1, 0)
if err != nil {
return err
}
namespaceIDs := []int64{}
namespaces := []*NamespaceWithListsAndTasks{}
2021-09-08 18:40:43 +02:00
listMap := make(map[int64]*ListWithTasksAndBuckets)
listIDs := []int64{}
for _, n := range namspaces.([]*NamespaceWithLists) {
if n.ID < 1 {
// Don't include filters
continue
}
nn := &NamespaceWithListsAndTasks{
Namespace: n.Namespace,
Lists: []*ListWithTasksAndBuckets{},
}
for _, l := range n.Lists {
2021-09-08 18:40:43 +02:00
ll := &ListWithTasksAndBuckets{
List: *l,
BackgroundFileID: l.BackgroundFileID,
Tasks: []*TaskWithComments{},
2021-09-08 18:40:43 +02:00
}
nn.Lists = append(nn.Lists, ll)
listMap[l.ID] = ll
listIDs = append(listIDs, l.ID)
}
namespaceIDs = append(namespaceIDs, n.ID)
namespaces = append(namespaces, nn)
}
if len(namespaceIDs) == 0 {
return nil
}
// Get all lists
lists, err := getListsForNamespaces(s, namespaceIDs, true)
if err != nil {
return err
}
tasks, _, _, err := getTasksForLists(s, lists, u, &taskOptions{
page: 0,
perPage: -1,
})
if err != nil {
return err
}
taskMap := make(map[int64]*TaskWithComments, len(tasks))
for _, t := range tasks {
taskMap[t.ID] = &TaskWithComments{
Task: *t,
}
if _, exists := listMap[t.ListID]; !exists {
2021-09-07 19:09:44 +02:00
log.Debugf("[User Data Export] List %d does not exist for task %d, omitting", t.ListID, t.ID)
continue
}
listMap[t.ListID].Tasks = append(listMap[t.ListID].Tasks, taskMap[t.ID])
}
comments := []*TaskComment{}
err = s.
Join("LEFT", "tasks", "tasks.id = task_comments.task_id").
In("tasks.list_id", listIDs).
Find(&comments)
if err != nil {
return
}
for _, c := range comments {
if _, exists := taskMap[c.TaskID]; !exists {
2021-09-07 19:09:44 +02:00
log.Debugf("[User Data Export] Task %d does not exist for comment %d, omitting", c.TaskID, c.ID)
continue
}
taskMap[c.TaskID].Comments = append(taskMap[c.TaskID].Comments, c)
}
buckets := []*Bucket{}
err = s.In("list_id", listIDs).Find(&buckets)
if err != nil {
return
}
for _, b := range buckets {
if _, exists := listMap[b.ListID]; !exists {
2021-09-07 19:09:44 +02:00
log.Debugf("[User Data Export] List %d does not exist for bucket %d, omitting", b.ListID, b.ID)
continue
}
listMap[b.ListID].Buckets = append(listMap[b.ListID].Buckets, b)
}
data, err := json.Marshal(namespaces)
if err != nil {
return err
}
return utils.WriteBytesToZip("data.json", data, wr)
}
func exportTaskAttachments(s *xorm.Session, u *user.User, wr *zip.Writer) (err error) {
lists, _, _, err := getRawListsForUser(
s,
&listOptions{
user: u,
page: -1,
},
)
if err != nil {
return err
}
tasks, _, _, err := getRawTasksForLists(s, lists, u, &taskOptions{page: -1})
if err != nil {
return err
}
taskIDs := []int64{}
for _, t := range tasks {
taskIDs = append(taskIDs, t.ID)
}
tas, err := getTaskAttachmentsByTaskIDs(s, taskIDs)
if err != nil {
return err
}
fs := make(map[int64]io.ReadCloser)
for _, ta := range tas {
if err := ta.File.LoadFileByID(); err != nil {
return err
}
fs[ta.FileID] = ta.File.File
}
return utils.WriteFilesToZip(fs, wr)
}
func exportSavedFilters(s *xorm.Session, u *user.User, wr *zip.Writer) (err error) {
filters, err := getSavedFiltersForUser(s, u)
if err != nil {
return err
}
data, err := json.Marshal(filters)
if err != nil {
return err
}
return utils.WriteBytesToZip("filters.json", data, wr)
}
func exportListBackgrounds(s *xorm.Session, u *user.User, wr *zip.Writer) (err error) {
lists, _, _, err := getRawListsForUser(
s,
&listOptions{
user: u,
page: -1,
},
)
if err != nil {
return err
}
fs := make(map[int64]io.ReadCloser)
for _, l := range lists {
if l.BackgroundFileID == 0 {
continue
}
bgFile := &files.File{
ID: l.BackgroundFileID,
}
err = bgFile.LoadFileByID()
if err != nil {
return
}
fs[l.BackgroundFileID] = bgFile.File
}
return utils.WriteFilesToZip(fs, wr)
}
func RegisterOldExportCleanupCron() {
const logPrefix = "[User Export Cleanup Cron] "
err := cron.Schedule("0 * * * *", func() {
s := db.NewSession()
defer s.Close()
users := []*user.User{}
err := s.Where("export_file_id IS NOT NULL AND export_file_id != ?", 0).Find(&users)
if err != nil {
log.Errorf(logPrefix+"Could not get users with export files: %s", err)
return
}
fileIDs := []int64{}
for _, u := range users {
fileIDs = append(fileIDs, u.ExportFileID)
}
fs := []*files.File{}
err = s.Where("created < ?", time.Now().Add(-time.Hour*24*7)).In("id", fileIDs).Find(&fs)
if err != nil {
log.Errorf(logPrefix+"Could not get users with export files: %s", err)
return
}
if len(fs) == 0 {
return
}
log.Debugf(logPrefix+"Removing %d old user data exports...", len(fs))
for _, f := range fs {
err = f.Delete()
if err != nil {
log.Errorf(logPrefix+"Could not remove user export file %d: %s", f.ID, err)
return
}
}
_, err = s.In("export_file_id", fileIDs).Cols("export_file_id").Update(&user.User{})
if err != nil {
log.Errorf(logPrefix+"Could not update user export file state: %s", err)
return
}
log.Debugf(logPrefix+"Removed %d old user data exports...", len(fs))
})
if err != nil {
log.Fatalf("Could not old export cleanup cron: %s", err)
}
}