vikunja-api/pkg/modules/migration/wunderlist/wunderlist.go

498 lines
15 KiB
Go
Raw Normal View History

2020-02-07 17:27:45 +01:00
// Vikunja is a to-do list application to facilitate your life.
// Copyright 2018-2020 Vikunja and contributors. All rights reserved.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package wunderlist
import (
"bytes"
"code.vikunja.io/api/pkg/config"
"code.vikunja.io/api/pkg/files"
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/models"
"code.vikunja.io/api/pkg/modules/migration"
"code.vikunja.io/api/pkg/timeutil"
"code.vikunja.io/api/pkg/user"
"code.vikunja.io/api/pkg/utils"
"encoding/json"
"fmt"
"net/http"
"net/url"
"strconv"
"time"
)
// Migration represents the implementation of the migration for wunderlist
type Migration struct {
// Code is the code used to get a user api token
Code string `query:"code" json:"code"`
}
// This represents all necessary fields for getting an api token for the wunderlist api from a code
type wunderlistAuthRequest struct {
ClientID string `json:"client_id"`
ClientSecret string `json:"client_secret"`
Code string `json:"code"`
}
type wunderlistAuthToken struct {
AccessToken string `json:"access_token"`
}
type task struct {
ID int `json:"id"`
AssigneeID int `json:"assignee_id"`
CreatedAt time.Time `json:"created_at"`
CreatedByID int `json:"created_by_id"`
DueDate string `json:"due_date"`
ListID int `json:"list_id"`
Revision int `json:"revision"`
Starred bool `json:"starred"`
Title string `json:"title"`
Completed bool `json:"completed"`
CompletedAt time.Time `json:"completed_at"`
}
type list struct {
ID int `json:"id"`
CreatedAt time.Time `json:"created_at"`
Title string `json:"title"`
ListType string `json:"list_type"`
Type string `json:"type"`
Revision int `json:"revision"`
Migrated bool `json:"-"`
}
type folder struct {
ID int `json:"id"`
Title string `json:"title"`
ListIds []int `json:"list_ids"`
CreatedAt time.Time `json:"created_at"`
CreatedByRequestID string `json:"created_by_request_id"`
UpdatedAt time.Time `json:"updated_at"`
Type string `json:"type"`
Revision int `json:"revision"`
}
type note struct {
ID int `json:"id"`
TaskID int `json:"task_id"`
Content string `json:"content"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
Revision int `json:"revision"`
}
type file struct {
ID int `json:"id"`
URL string `json:"url"`
TaskID int `json:"task_id"`
ListID int `json:"list_id"`
UserID int `json:"user_id"`
FileName string `json:"file_name"`
ContentType string `json:"content_type"`
FileSize int `json:"file_size"`
LocalCreatedAt time.Time `json:"local_created_at"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
Type string `json:"type"`
Revision int `json:"revision"`
}
type reminder struct {
ID int `json:"id"`
Date time.Time `json:"date"`
TaskID int `json:"task_id"`
Revision int `json:"revision"`
Type string `json:"type"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
type subtask struct {
ID int `json:"id"`
TaskID int `json:"task_id"`
CreatedAt time.Time `json:"created_at"`
CreatedByID int `json:"created_by_id"`
Revision int `json:"revision"`
Title string `json:"title"`
}
type wunderlistContents struct {
tasks []*task
lists []*list
folders []*folder
notes []*note
files []*file
reminders []*reminder
subtasks []*subtask
}
func convertListForFolder(listID int, list *list, content *wunderlistContents) (*models.List, error) {
l := &models.List{
Title: list.Title,
Created: timeutil.FromTime(list.CreatedAt),
}
// Find all tasks belonging to this list and put them in
for _, t := range content.tasks {
if t.ListID == listID {
newTask := &models.Task{
Text: t.Title,
Created: timeutil.FromTime(t.CreatedAt),
Done: t.Completed,
}
// Set Done At
if newTask.Done {
newTask.DoneAt = timeutil.FromTime(t.CompletedAt)
}
// Parse the due date
if t.DueDate != "" {
dueDate, err := time.Parse("2006-01-02", t.DueDate)
if err != nil {
return nil, err
}
newTask.DueDate = timeutil.FromTime(dueDate)
}
// Find related notes
for _, n := range content.notes {
if n.TaskID == t.ID {
newTask.Description = n.Content
}
}
// Attachments
for _, f := range content.files {
if f.TaskID == t.ID {
// Download the attachment and put it in the file
resp, err := http.Get(f.URL)
if err != nil {
return nil, err
}
defer resp.Body.Close()
buf := &bytes.Buffer{}
_, err = buf.ReadFrom(resp.Body)
if err != nil {
return nil, err
}
newTask.Attachments = append(newTask.Attachments, &models.TaskAttachment{
File: &files.File{
Name: f.FileName,
Mime: f.ContentType,
Size: uint64(f.FileSize),
Created: f.CreatedAt,
CreatedUnix: timeutil.FromTime(f.CreatedAt),
// We directly pass the file contents here to have a way to link the attachment to the file later.
// Because we don't have an ID for our task at this point of the migration, we cannot just throw all
// attachments in a slice and do the work of downloading and properly storing them later.
FileContent: buf.Bytes(),
},
Created: timeutil.FromTime(f.CreatedAt),
})
}
}
// Subtasks
for _, s := range content.subtasks {
if s.TaskID == t.ID {
if newTask.RelatedTasks[models.RelationKindSubtask] == nil {
newTask.RelatedTasks = make(models.RelatedTaskMap)
}
newTask.RelatedTasks[models.RelationKindSubtask] = append(newTask.RelatedTasks[models.RelationKindSubtask], &models.Task{
Text: s.Title,
})
}
}
// Reminders
for _, r := range content.reminders {
if r.TaskID == t.ID {
newTask.Reminders = append(newTask.Reminders, timeutil.FromTime(r.Date))
}
}
l.Tasks = append(l.Tasks, newTask)
}
}
return l, nil
}
func convertWunderlistToVikunja(content *wunderlistContents) (fullVikunjaHierachie []*models.NamespaceWithLists, err error) {
// Make a map from the list with the key being list id for easier handling
listMap := make(map[int]*list, len(content.lists))
for _, l := range content.lists {
listMap[l.ID] = l
}
// First, we look through all folders and create namespaces for them.
for _, folder := range content.folders {
namespace := &models.NamespaceWithLists{
Namespace: models.Namespace{
Name: folder.Title,
Created: timeutil.FromTime(folder.CreatedAt),
Updated: timeutil.FromTime(folder.UpdatedAt),
},
}
// Then find all lists for that folder
for _, listID := range folder.ListIds {
if list, exists := listMap[listID]; exists {
l, err := convertListForFolder(listID, list, content)
if err != nil {
return nil, err
}
namespace.Lists = append(namespace.Lists, l)
// And mark the list as migrated so we don't iterate over it again
list.Migrated = true
}
}
// And then finally put the namespace (which now has all the details) back in the full array.
fullVikunjaHierachie = append(fullVikunjaHierachie, namespace)
}
// At the end, loop over all lists which don't belong to a namespace and put them in a default namespace
if len(listMap) > 0 {
newNamespace := &models.NamespaceWithLists{
Namespace: models.Namespace{
Name: "Migrated from wunderlist",
},
}
for _, list := range listMap {
if list.Migrated {
continue
}
l, err := convertListForFolder(list.ID, list, content)
if err != nil {
return nil, err
}
newNamespace.Lists = append(newNamespace.Lists, l)
}
fullVikunjaHierachie = append(fullVikunjaHierachie, newNamespace)
}
return
}
func makeAuthGetRequest(token *wunderlistAuthToken, urlPart string, v interface{}, urlParams url.Values) error {
req, err := http.NewRequest(http.MethodGet, "https://a.wunderlist.com/api/v1/"+urlPart, nil)
if err != nil {
return err
}
req.Header.Set("X-Access-Token", token.AccessToken)
req.Header.Set("X-Client-ID", config.MigrationWunderlistClientID.GetString())
req.URL.RawQuery = urlParams.Encode()
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
buf := &bytes.Buffer{}
_, err = buf.ReadFrom(resp.Body)
if err != nil {
return err
}
if resp.StatusCode > 399 {
return fmt.Errorf("wunderlist API Error: Status Code: %d, Response was: %s", resp.StatusCode, buf.String())
}
// If the response is an empty json array, we need to exit here, otherwise this breaks the json parser since it
// expects a null for an empty slice
str := buf.String()
if str == "[]" {
return nil
}
return json.Unmarshal(buf.Bytes(), v)
}
// Migrate migrates a user's wunderlist lists, tasks, etc.
// @Summary Migrate all lists, tasks etc. from wunderlist
// @Description Migrates all folders, lists, tasks, notes, reminders, subtasks and files from wunderlist to vikunja.
// @tags migration
// @Accept json
// @Produce json
// @Security JWTKeyAuth
// @Param migrationCode body wunderlist.Migration true "The auth code previously obtained from the auth url. See the docs for /migration/wunderlist/auth."
// @Success 200 {object} models.Message "A message telling you everything was migrated successfully."
// @Failure 500 {object} models.Message "Internal server error"
// @Router /migration/wunderlist/migrate [post]
func (w *Migration) Migrate(user *user.User) (err error) {
log.Debugf("[Wunderlist migration] Starting wunderlist migration for user %d", user.ID)
// Struct init
wContent := &wunderlistContents{
tasks: []*task{},
lists: []*list{},
folders: []*folder{},
notes: []*note{},
files: []*file{},
reminders: []*reminder{},
subtasks: []*subtask{},
}
// 0. Get api token from oauth user token
authRequest := wunderlistAuthRequest{
ClientID: config.MigrationWunderlistClientID.GetString(),
ClientSecret: config.MigrationWunderlistClientSecret.GetString(),
Code: w.Code,
}
jsonAuth, err := json.Marshal(authRequest)
if err != nil {
return
}
resp, err := http.Post("https://www.wunderlist.com/oauth/access_token", "application/json", bytes.NewBuffer(jsonAuth))
if err != nil {
return
}
authToken := &wunderlistAuthToken{}
err = json.NewDecoder(resp.Body).Decode(authToken)
if err != nil {
return
}
log.Debugf("[Wunderlist migration] Start getting all data from wunderlist for user %d", user.ID)
// 1. Get all folders
err = makeAuthGetRequest(authToken, "folders", &wContent.folders, nil)
if err != nil {
return
}
// 2. Get all lists
err = makeAuthGetRequest(authToken, "lists", &wContent.lists, nil)
if err != nil {
return
}
for _, l := range wContent.lists {
listQueryParam := url.Values{"list_id": []string{strconv.Itoa(l.ID)}}
// 3. Get all tasks for each list
tasks := []*task{}
err = makeAuthGetRequest(authToken, "tasks", &tasks, listQueryParam)
if err != nil {
return
}
wContent.tasks = append(wContent.tasks, tasks...)
// 3. Get all done tasks for each list
doneTasks := []*task{}
err = makeAuthGetRequest(authToken, "tasks", &doneTasks, url.Values{"list_id": []string{strconv.Itoa(l.ID)}, "completed": []string{"true"}})
if err != nil {
return
}
wContent.tasks = append(wContent.tasks, doneTasks...)
// 4. Get all notes for all lists
notes := []*note{}
err = makeAuthGetRequest(authToken, "notes", &notes, listQueryParam)
if err != nil {
return
}
wContent.notes = append(wContent.notes, notes...)
// 5. Get all files for all lists
fils := []*file{}
err = makeAuthGetRequest(authToken, "files", &fils, listQueryParam)
if err != nil {
return
}
wContent.files = append(wContent.files, fils...)
// 6. Get all reminders for all lists
reminders := []*reminder{}
err = makeAuthGetRequest(authToken, "reminders", &reminders, listQueryParam)
if err != nil {
return
}
wContent.reminders = append(wContent.reminders, reminders...)
// 7. Get all subtasks for all lists
subtasks := []*subtask{}
err = makeAuthGetRequest(authToken, "subtasks", &subtasks, listQueryParam)
if err != nil {
return
}
wContent.subtasks = append(wContent.subtasks, subtasks...)
}
log.Debugf("[Wunderlist migration] Got all data from wunderlist for user %d", user.ID)
log.Debugf("[Wunderlist migration] Migrating data to vikunja format for user %d", user.ID)
// Convert + Insert everything
fullVikunjaHierachie, err := convertWunderlistToVikunja(wContent)
if err != nil {
return
}
log.Debugf("[Wunderlist migration] Done migrating data to vikunja format for user %d", user.ID)
log.Debugf("[Wunderlist migration] Insert data into db for user %d", user.ID)
err = migration.InsertFromStructure(fullVikunjaHierachie, user)
log.Debugf("[Wunderlist migration] Done inserting data into db for user %d", user.ID)
log.Debugf("[Wunderlist migration] Wunderlist migration for user %d done", user.ID)
return err
}
// AuthURL returns the url users need to authenticate against
// @Summary Get the auth url from wunderlist
// @Description Returns the auth url where the user needs to get its auth code. This code can then be used to migrate everything from wunderlist to Vikunja.
// @tags migration
// @Produce json
// @Security JWTKeyAuth
// @Success 200 {object} handler.AuthURL "The auth url."
// @Failure 500 {object} models.Message "Internal server error"
// @Router /migration/wunderlist/auth [get]
func (w *Migration) AuthURL() string {
return "https://www.wunderlist.com/oauth/authorize?client_id=" +
config.MigrationWunderlistClientID.GetString() +
"&redirect_uri=" +
config.MigrationWunderlistRedirectURL.GetString() +
"&state=" + utils.MakeRandomString(32)
}
// Name is used to get the name of the wunderlist migration
// @Summary Get migration status
// @Description Returns if the current user already did the migation or not. This is useful to show a confirmation message in the frontend if the user is trying to do the same migration again.
// @tags migration
// @Produce json
// @Security JWTKeyAuth
// @Success 200 {object} migration.Status "The migration status"
// @Failure 500 {object} models.Message "Internal server error"
// @Router /migration/wunderlist/status [get]
func (w *Migration) Name() string {
return "wunderlist"
}