521 lines
16 KiB
Go
521 lines
16 KiB
Go
// Vikunja is a to-do list application to facilitate your life.
|
|
// Copyright 2018-2020 Vikunja and contributors. All rights reserved.
|
|
//
|
|
// This program is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU General Public License as published by
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU General Public License
|
|
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
|
|
package todoist
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
"code.vikunja.io/api/pkg/config"
|
|
"code.vikunja.io/api/pkg/files"
|
|
"code.vikunja.io/api/pkg/log"
|
|
"code.vikunja.io/api/pkg/models"
|
|
"code.vikunja.io/api/pkg/modules/migration"
|
|
"code.vikunja.io/api/pkg/user"
|
|
"code.vikunja.io/api/pkg/utils"
|
|
)
|
|
|
|
// Migration is the todoist migration struct
|
|
type Migration struct {
|
|
Code string `json:"code"`
|
|
}
|
|
|
|
type apiTokenResponse struct {
|
|
AccessToken string `json:"access_token"`
|
|
TokenType string `json:"token_type"`
|
|
}
|
|
|
|
type label struct {
|
|
ID int64 `json:"id"`
|
|
Name string `json:"name"`
|
|
Color int64 `json:"color"`
|
|
ItemOrder int64 `json:"item_order"`
|
|
IsDeleted int64 `json:"is_deleted"`
|
|
IsFavorite int64 `json:"is_favorite"`
|
|
}
|
|
|
|
type project struct {
|
|
ID int64 `json:"id"`
|
|
LegacyID int64 `json:"legacy_id"`
|
|
Name string `json:"name"`
|
|
Color int64 `json:"color"`
|
|
ParentID int64 `json:"parent_id"`
|
|
ChildOrder int64 `json:"child_order"`
|
|
Collapsed int64 `json:"collapsed"`
|
|
Shared bool `json:"shared"`
|
|
LegacyParentID int64 `json:"legacy_parent_id"`
|
|
SyncID int64 `json:"sync_id"`
|
|
IsDeleted int64 `json:"is_deleted"`
|
|
IsArchived int64 `json:"is_archived"`
|
|
IsFavorite int64 `json:"is_favorite"`
|
|
}
|
|
|
|
type dueDate struct {
|
|
Date string `json:"date"`
|
|
Timezone interface{} `json:"timezone"`
|
|
String string `json:"string"`
|
|
Lang string `json:"lang"`
|
|
IsRecurring bool `json:"is_recurring"`
|
|
}
|
|
|
|
type item struct {
|
|
ID int64 `json:"id"`
|
|
LegacyID int64 `json:"legacy_id"`
|
|
UserID int64 `json:"user_id"`
|
|
ProjectID int64 `json:"project_id"`
|
|
LegacyProjectID int64 `json:"legacy_project_id"`
|
|
Content string `json:"content"`
|
|
Priority int64 `json:"priority"`
|
|
Due *dueDate `json:"due"`
|
|
ParentID int64 `json:"parent_id"`
|
|
LegacyParentID int64 `json:"legacy_parent_id"`
|
|
ChildOrder int64 `json:"child_order"`
|
|
SectionID int64 `json:"section_id"`
|
|
DayOrder int64 `json:"day_order"`
|
|
Collapsed int64 `json:"collapsed"`
|
|
Children interface{} `json:"children"`
|
|
Labels []int64 `json:"labels"`
|
|
AddedByUID int64 `json:"added_by_uid"`
|
|
AssignedByUID int64 `json:"assigned_by_uid"`
|
|
ResponsibleUID int64 `json:"responsible_uid"`
|
|
Checked int64 `json:"checked"`
|
|
InHistory int64 `json:"in_history"`
|
|
IsDeleted int64 `json:"is_deleted"`
|
|
DateAdded time.Time `json:"date_added"`
|
|
HasMoreNotes bool `json:"has_more_notes"`
|
|
DateCompleted time.Time `json:"date_completed"`
|
|
}
|
|
|
|
type fileAttachment struct {
|
|
FileType string `json:"file_type"`
|
|
FileName string `json:"file_name"`
|
|
FileSize int64 `json:"file_size"`
|
|
FileURL string `json:"file_url"`
|
|
UploadState string `json:"upload_state"`
|
|
}
|
|
|
|
type note struct {
|
|
ID int64 `json:"id"`
|
|
LegacyID int64 `json:"legacy_id"`
|
|
PostedUID int64 `json:"posted_uid"`
|
|
ProjectID int64 `json:"project_id"`
|
|
LegacyProjectID int64 `json:"legacy_project_id"`
|
|
ItemID int64 `json:"item_id"`
|
|
LegacyItemID int64 `json:"legacy_item_id"`
|
|
Content string `json:"content"`
|
|
FileAttachment *fileAttachment `json:"file_attachment"`
|
|
UidsToNotify []int64 `json:"uids_to_notify"`
|
|
IsDeleted int64 `json:"is_deleted"`
|
|
Posted time.Time `json:"posted"`
|
|
}
|
|
|
|
type projectNote struct {
|
|
Content string `json:"content"`
|
|
FileAttachment *fileAttachment `json:"file_attachment"`
|
|
ID int64 `json:"id"`
|
|
IsDeleted int64 `json:"is_deleted"`
|
|
Posted time.Time `json:"posted"`
|
|
PostedUID int64 `json:"posted_uid"`
|
|
ProjectID int64 `json:"project_id"`
|
|
UidsToNotify []int64 `json:"uids_to_notify"`
|
|
}
|
|
|
|
type reminder struct {
|
|
ID int64 `json:"id"`
|
|
NotifyUID int64 `json:"notify_uid"`
|
|
ItemID int64 `json:"item_id"`
|
|
Service string `json:"service"`
|
|
Type string `json:"type"`
|
|
Due *dueDate `json:"due"`
|
|
MmOffset int64 `json:"mm_offset"`
|
|
IsDeleted int64 `json:"is_deleted"`
|
|
}
|
|
|
|
type sync struct {
|
|
Projects []*project `json:"projects"`
|
|
Items []*item `json:"items"`
|
|
Labels []*label `json:"labels"`
|
|
Notes []*note `json:"notes"`
|
|
ProjectNotes []*projectNote `json:"project_notes"`
|
|
Reminders []*reminder `json:"reminders"`
|
|
}
|
|
|
|
var todoistColors = map[int64]string{}
|
|
|
|
func init() {
|
|
todoistColors = make(map[int64]string, 19)
|
|
// The todoists colors are static, taken from https://developer.todoist.com/sync/v8/#colors
|
|
todoistColors = map[int64]string{
|
|
30: "b8256f",
|
|
31: "db4035",
|
|
32: "ff9933",
|
|
33: "fad000",
|
|
34: "afb83b",
|
|
35: "7ecc49",
|
|
36: "299438",
|
|
37: "6accbc",
|
|
38: "158fad",
|
|
39: "14aaf5",
|
|
40: "96c3eb",
|
|
41: "4073ff",
|
|
42: "884dff",
|
|
43: "af38eb",
|
|
44: "eb96eb",
|
|
45: "e05194",
|
|
46: "ff8d85",
|
|
47: "808080",
|
|
48: "b8b8b8",
|
|
49: "ccac93",
|
|
}
|
|
}
|
|
|
|
// Name is used to get the name of the todoist migration - we're using the docs here to annotate the status route.
|
|
// @Summary Get migration status
|
|
// @Description Returns if the current user already did the migation or not. This is useful to show a confirmation message in the frontend if the user is trying to do the same migration again.
|
|
// @tags migration
|
|
// @Produce json
|
|
// @Security JWTKeyAuth
|
|
// @Success 200 {object} migration.Status "The migration status"
|
|
// @Failure 500 {object} models.Message "Internal server error"
|
|
// @Router /migration/todoist/status [get]
|
|
func (m *Migration) Name() string {
|
|
return "todoist"
|
|
}
|
|
|
|
// AuthURL returns the url users need to authenticate against
|
|
// @Summary Get the auth url from todoist
|
|
// @Description Returns the auth url where the user needs to get its auth code. This code can then be used to migrate everything from todoist to Vikunja.
|
|
// @tags migration
|
|
// @Produce json
|
|
// @Security JWTKeyAuth
|
|
// @Success 200 {object} handler.AuthURL "The auth url."
|
|
// @Failure 500 {object} models.Message "Internal server error"
|
|
// @Router /migration/todoist/auth [get]
|
|
func (m *Migration) AuthURL() string {
|
|
return "https://todoist.com/oauth/authorize" +
|
|
"?client_id=" + config.MigrationTodoistClientID.GetString() +
|
|
"&scope=data:read" +
|
|
"&state=" + utils.MakeRandomString(32)
|
|
}
|
|
|
|
func doPost(url string, form url.Values) (resp *http.Response, err error) {
|
|
req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, url, strings.NewReader(form.Encode()))
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
|
|
hc := http.Client{}
|
|
return hc.Do(req)
|
|
}
|
|
|
|
func convertTodoistToVikunja(sync *sync) (fullVikunjaHierachie []*models.NamespaceWithLists, err error) {
|
|
|
|
newNamespace := &models.NamespaceWithLists{
|
|
Namespace: models.Namespace{
|
|
Title: "Migrated from todoist",
|
|
},
|
|
}
|
|
|
|
// A map for all vikunja lists with the project id they're coming from as key
|
|
lists := make(map[int64]*models.List, len(sync.Projects))
|
|
|
|
// A map for all vikunja tasks with the todoist task id as key to find them easily and add more data
|
|
tasks := make(map[int64]*models.Task, len(sync.Items))
|
|
|
|
// A map for all vikunja labels with the todoist id as key to find them easier
|
|
labels := make(map[int64]*models.Label, len(sync.Labels))
|
|
|
|
for _, p := range sync.Projects {
|
|
list := &models.List{
|
|
Title: p.Name,
|
|
HexColor: todoistColors[p.Color],
|
|
IsArchived: p.IsArchived == 1,
|
|
}
|
|
|
|
lists[p.ID] = list
|
|
|
|
newNamespace.Lists = append(newNamespace.Lists, list)
|
|
}
|
|
|
|
for _, label := range sync.Labels {
|
|
labels[label.ID] = &models.Label{
|
|
Title: label.Name,
|
|
HexColor: todoistColors[label.Color],
|
|
}
|
|
}
|
|
|
|
for _, i := range sync.Items {
|
|
task := &models.Task{
|
|
Title: i.Content,
|
|
Created: i.DateAdded.In(config.GetTimeZone()),
|
|
Done: i.Checked == 1,
|
|
}
|
|
|
|
// Only try to parse the task done at date if the task is actually done
|
|
// Sometimes weired things happen if we try to parse nil dates.
|
|
if task.Done {
|
|
task.DoneAt = i.DateCompleted.In(config.GetTimeZone())
|
|
}
|
|
|
|
// Todoist priorities only range from 1 (lowest) and max 4 (highest), so we need to make slight adjustments
|
|
if i.Priority > 1 {
|
|
task.Priority = i.Priority
|
|
}
|
|
|
|
// Put the due date together
|
|
if i.Due != nil {
|
|
dueDate, err := time.Parse("2006-01-02", i.Due.Date)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
task.DueDate = dueDate.In(config.GetTimeZone())
|
|
}
|
|
|
|
// Put all labels together from earlier
|
|
for _, lID := range i.Labels {
|
|
task.Labels = append(task.Labels, labels[lID])
|
|
}
|
|
|
|
tasks[i.ID] = task
|
|
|
|
lists[i.ProjectID].Tasks = append(lists[i.ProjectID].Tasks, task)
|
|
}
|
|
|
|
// If the parenId of a task is not 0, create a task relation
|
|
// We're looping again here to make sure we have seem all tasks before and have them in our map
|
|
for _, i := range sync.Items {
|
|
if i.ParentID == 0 {
|
|
continue
|
|
}
|
|
|
|
if _, exists := tasks[i.ParentID]; !exists {
|
|
log.Debugf("[Todoist Migration] Could not find task %d in tasks map while trying to get resolve subtasks for task %d", i.ParentID, i.ID)
|
|
continue
|
|
}
|
|
|
|
// Prevent all those nil errors
|
|
if tasks[i.ParentID].RelatedTasks == nil {
|
|
tasks[i.ParentID].RelatedTasks = make(models.RelatedTaskMap)
|
|
}
|
|
|
|
tasks[i.ParentID].RelatedTasks[models.RelationKindSubtask] = append(tasks[i.ParentID].RelatedTasks[models.RelationKindSubtask], tasks[i.ID])
|
|
|
|
// Remove the task from the top level structure, otherwise it is added twice
|
|
outer:
|
|
for _, list := range lists {
|
|
for in, t := range list.Tasks {
|
|
if t == tasks[i.ID] {
|
|
list.Tasks = append(list.Tasks[:in], list.Tasks[in+1:]...)
|
|
break outer
|
|
}
|
|
}
|
|
}
|
|
delete(tasks, i.ID)
|
|
}
|
|
|
|
// Task Notes -> Task Descriptions
|
|
// FIXME: Should be comments
|
|
for _, n := range sync.Notes {
|
|
if _, exists := tasks[n.ItemID]; !exists {
|
|
log.Debugf("[Todoist Migration] Could not find task %d for note %d", n.ItemID, n.ID)
|
|
continue
|
|
}
|
|
|
|
if tasks[n.ItemID].Description != "" {
|
|
tasks[n.ItemID].Description += "\n"
|
|
}
|
|
tasks[n.ItemID].Description += n.Content
|
|
|
|
if n.FileAttachment == nil {
|
|
continue
|
|
}
|
|
|
|
// Only add the attachment if there's something to download
|
|
if len(n.FileAttachment.FileURL) > 0 {
|
|
// Download the attachment and put it in the file
|
|
req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, n.FileAttachment.FileURL, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
hc := http.Client{}
|
|
resp, err := hc.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
buf := &bytes.Buffer{}
|
|
_, err = buf.ReadFrom(resp.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
tasks[n.ItemID].Attachments = append(tasks[n.ItemID].Attachments, &models.TaskAttachment{
|
|
File: &files.File{
|
|
Name: n.FileAttachment.FileName,
|
|
Mime: n.FileAttachment.FileType,
|
|
Size: uint64(n.FileAttachment.FileSize),
|
|
Created: n.Posted,
|
|
// We directly pass the file contents here to have a way to link the attachment to the file later.
|
|
// Because we don't have an ID for our task at this point of the migration, we cannot just throw all
|
|
// attachments in a slice and do the work of downloading and properly storing them later.
|
|
FileContent: buf.Bytes(),
|
|
},
|
|
Created: n.Posted,
|
|
})
|
|
}
|
|
}
|
|
|
|
// Project Notes -> List Descriptions
|
|
for _, pn := range sync.ProjectNotes {
|
|
if lists[pn.ProjectID].Description != "" {
|
|
lists[pn.ProjectID].Description += "\n"
|
|
}
|
|
|
|
lists[pn.ProjectID].Description += pn.Content
|
|
}
|
|
|
|
// Reminders -> vikunja reminders
|
|
for _, r := range sync.Reminders {
|
|
if r.Due == nil {
|
|
continue
|
|
}
|
|
|
|
if _, exists := tasks[r.ItemID]; !exists {
|
|
log.Debugf("Could not find task %d for reminder %d while trying to resolve reminders", r.ItemID, r.ID)
|
|
}
|
|
|
|
var err error
|
|
var date time.Time
|
|
date, err = time.Parse("2006-01-02T15:04:05Z", r.Due.Date)
|
|
if err != nil {
|
|
date, err = time.Parse("2006-01-02T15:04:05", r.Due.Date)
|
|
}
|
|
if err != nil {
|
|
date, err = time.Parse("2006-01-02", r.Due.Date)
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
tasks[r.ItemID].Reminders = append(tasks[r.ItemID].Reminders, date.In(config.GetTimeZone()))
|
|
}
|
|
|
|
return []*models.NamespaceWithLists{
|
|
newNamespace,
|
|
}, err
|
|
}
|
|
|
|
func getAccessTokenFromAuthToken(authToken string) (accessToken string, err error) {
|
|
|
|
form := url.Values{
|
|
"client_id": []string{config.MigrationTodoistClientID.GetString()},
|
|
"client_secret": []string{config.MigrationTodoistClientSecret.GetString()},
|
|
"code": []string{authToken},
|
|
"redirect_uri": []string{config.MigrationTodoistRedirectURL.GetString()},
|
|
}
|
|
resp, err := doPost("https://todoist.com/oauth/access_token", form)
|
|
if err != nil {
|
|
return
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode > 399 {
|
|
buf := &bytes.Buffer{}
|
|
_, _ = buf.ReadFrom(resp.Body)
|
|
return "", fmt.Errorf("got http status %d while trying to get token, error was %s", resp.StatusCode, buf.String())
|
|
}
|
|
|
|
token := &apiTokenResponse{}
|
|
err = json.NewDecoder(resp.Body).Decode(token)
|
|
return token.AccessToken, err
|
|
}
|
|
|
|
// Migrate gets all tasks from todoist for a user and puts them into vikunja
|
|
// @Summary Migrate all lists, tasks etc. from todoist
|
|
// @Description Migrates all projects, tasks, notes, reminders, subtasks and files from todoist to vikunja.
|
|
// @tags migration
|
|
// @Accept json
|
|
// @Produce json
|
|
// @Security JWTKeyAuth
|
|
// @Param migrationCode body todoist.Migration true "The auth code previously obtained from the auth url. See the docs for /migration/todoist/auth."
|
|
// @Success 200 {object} models.Message "A message telling you everything was migrated successfully."
|
|
// @Failure 500 {object} models.Message "Internal server error"
|
|
// @Router /migration/todoist/migrate [post]
|
|
func (m *Migration) Migrate(u *user.User) (err error) {
|
|
|
|
log.Debugf("[Todoist Migration] Starting migration for user %d", u.ID)
|
|
|
|
// 0. Get an api token from the obtained auth token
|
|
token, err := getAccessTokenFromAuthToken(m.Code)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
if token == "" {
|
|
log.Debugf("[Todoist Migration] Could not get token")
|
|
return
|
|
}
|
|
|
|
log.Debugf("[Todoist Migration] Got user token for user %d", u.ID)
|
|
log.Debugf("[Todoist Migration] Getting todoist data for user %d", u.ID)
|
|
|
|
// Get everything with the sync api
|
|
form := url.Values{
|
|
"token": []string{token},
|
|
"sync_token": []string{"*"},
|
|
"resource_types": []string{"[\"all\"]"},
|
|
}
|
|
resp, err := doPost("https://api.todoist.com/sync/v8/sync", form)
|
|
if err != nil {
|
|
return
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
syncResponse := &sync{}
|
|
err = json.NewDecoder(resp.Body).Decode(syncResponse)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
log.Debugf("[Todoist Migration] Got all todoist user data for user %d", u.ID)
|
|
log.Debugf("[Todoist Migration] Start converting data for user %d", u.ID)
|
|
|
|
fullVikunjaHierachie, err := convertTodoistToVikunja(syncResponse)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
log.Debugf("[Todoist Migration] Done converting data for user %d", u.ID)
|
|
log.Debugf("[Todoist Migration] Start inserting data for user %d", u.ID)
|
|
|
|
err = migration.InsertFromStructure(fullVikunjaHierachie, u)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
log.Debugf("[Todoist Migration] Done inserting data for user %d", u.ID)
|
|
log.Debugf("[Todoist Migration] Todoist migration done for user %d", u.ID)
|
|
|
|
return nil
|
|
}
|