08205008e7
Fix query param name Add option to include null results when filtering Always set db time to gmt Fix null filter Fix timezone setting for todoist parsing Fix timezone setting for wunderlist parsing Fix import Fix caldav reminder parsing Use timezone from config Add error and test for invalid filter values Fix integration tests Remove task collection date hack Fix task filter Fix lint Fix tests and fixtures for date timezone stuff Properly set timezone Change fixtures time zone to gmt Set db timezone Set created and updated timestamps for all fixtures Fix lint Fix test fixtures Fix misspell Fix test fixtures Partially fix tests Remove timeutil package Remove adding _unix suffix hack Remove _unix suffix Move all timeutil.TimeStamp to time.Time Remove all Unix suffixes in field names Add better error messages when running migrations Make sure to not migrate 0 unix timestamps to 1970 iso dates Add migration script for sqlite Add converting sqlite values Convert 0 unix timestamps to null in postgres Convert 0 to null in timestamps Automatically rename _unix suffix Add all tables and columns for migration Fix sql migration query for mysql Fail with an error if trying to use an unsupported dbms Co-authored-by: kolaente <k@knt.li> Reviewed-on: https://kolaente.dev/vikunja/api/pulls/594
474 lines
15 KiB
Go
474 lines
15 KiB
Go
// Vikunja is a to-do list application to facilitate your life.
|
|
// Copyright 2018-2020 Vikunja and contributors. All rights reserved.
|
|
//
|
|
// This program is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU General Public License as published by
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU General Public License
|
|
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
|
|
package todoist
|
|
|
|
import (
|
|
"bytes"
|
|
"code.vikunja.io/api/pkg/config"
|
|
"code.vikunja.io/api/pkg/files"
|
|
"code.vikunja.io/api/pkg/log"
|
|
"code.vikunja.io/api/pkg/models"
|
|
"code.vikunja.io/api/pkg/modules/migration"
|
|
"code.vikunja.io/api/pkg/user"
|
|
"code.vikunja.io/api/pkg/utils"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Migration is the todoist migration struct
|
|
type Migration struct {
|
|
Code string `json:"code"`
|
|
}
|
|
|
|
type apiTokenResponse struct {
|
|
AccessToken string `json:"access_token"`
|
|
TokenType string `json:"token_type"`
|
|
}
|
|
|
|
type label struct {
|
|
ID int `json:"id"`
|
|
Name string `json:"name"`
|
|
Color int `json:"color"`
|
|
ItemOrder int `json:"item_order"`
|
|
IsDeleted int `json:"is_deleted"`
|
|
IsFavorite int `json:"is_favorite"`
|
|
}
|
|
|
|
type project struct {
|
|
ID int `json:"id"`
|
|
LegacyID int `json:"legacy_id"`
|
|
Name string `json:"name"`
|
|
Color int `json:"color"`
|
|
ParentID int `json:"parent_id"`
|
|
ChildOrder int `json:"child_order"`
|
|
Collapsed int `json:"collapsed"`
|
|
Shared bool `json:"shared"`
|
|
LegacyParentID int `json:"legacy_parent_id"`
|
|
SyncID int `json:"sync_id"`
|
|
IsDeleted int `json:"is_deleted"`
|
|
IsArchived int `json:"is_archived"`
|
|
IsFavorite int `json:"is_favorite"`
|
|
}
|
|
|
|
type dueDate struct {
|
|
Date string `json:"date"`
|
|
Timezone interface{} `json:"timezone"`
|
|
String string `json:"string"`
|
|
Lang string `json:"lang"`
|
|
IsRecurring bool `json:"is_recurring"`
|
|
}
|
|
|
|
type item struct {
|
|
ID int `json:"id"`
|
|
LegacyID int `json:"legacy_id"`
|
|
UserID int `json:"user_id"`
|
|
ProjectID int `json:"project_id"`
|
|
LegacyProjectID int `json:"legacy_project_id"`
|
|
Content string `json:"content"`
|
|
Priority int `json:"priority"`
|
|
Due *dueDate `json:"due"`
|
|
ParentID int `json:"parent_id"`
|
|
LegacyParentID int `json:"legacy_parent_id"`
|
|
ChildOrder int `json:"child_order"`
|
|
SectionID int `json:"section_id"`
|
|
DayOrder int `json:"day_order"`
|
|
Collapsed int `json:"collapsed"`
|
|
Children interface{} `json:"children"`
|
|
Labels []int `json:"labels"`
|
|
AddedByUID int `json:"added_by_uid"`
|
|
AssignedByUID int `json:"assigned_by_uid"`
|
|
ResponsibleUID int `json:"responsible_uid"`
|
|
Checked int `json:"checked"`
|
|
InHistory int `json:"in_history"`
|
|
IsDeleted int `json:"is_deleted"`
|
|
DateAdded time.Time `json:"date_added"`
|
|
HasMoreNotes bool `json:"has_more_notes"`
|
|
DateCompleted time.Time `json:"date_completed"`
|
|
}
|
|
|
|
type fileAttachment struct {
|
|
FileType string `json:"file_type"`
|
|
FileName string `json:"file_name"`
|
|
FileSize int `json:"file_size"`
|
|
FileURL string `json:"file_url"`
|
|
UploadState string `json:"upload_state"`
|
|
}
|
|
|
|
type note struct {
|
|
ID int `json:"id"`
|
|
LegacyID int `json:"legacy_id"`
|
|
PostedUID int `json:"posted_uid"`
|
|
ProjectID int `json:"project_id"`
|
|
LegacyProjectID int `json:"legacy_project_id"`
|
|
ItemID int `json:"item_id"`
|
|
LegacyItemID int `json:"legacy_item_id"`
|
|
Content string `json:"content"`
|
|
FileAttachment *fileAttachment `json:"file_attachment"`
|
|
UidsToNotify []int `json:"uids_to_notify"`
|
|
IsDeleted int `json:"is_deleted"`
|
|
Posted time.Time `json:"posted"`
|
|
}
|
|
|
|
type projectNote struct {
|
|
Content string `json:"content"`
|
|
FileAttachment *fileAttachment `json:"file_attachment"`
|
|
ID int64 `json:"id"`
|
|
IsDeleted int `json:"is_deleted"`
|
|
Posted time.Time `json:"posted"`
|
|
PostedUID int `json:"posted_uid"`
|
|
ProjectID int `json:"project_id"`
|
|
UidsToNotify []int `json:"uids_to_notify"`
|
|
}
|
|
|
|
type reminder struct {
|
|
ID int `json:"id"`
|
|
NotifyUID int `json:"notify_uid"`
|
|
ItemID int `json:"item_id"`
|
|
Service string `json:"service"`
|
|
Type string `json:"type"`
|
|
Due *dueDate `json:"due"`
|
|
MmOffset int `json:"mm_offset"`
|
|
IsDeleted int `json:"is_deleted"`
|
|
}
|
|
|
|
type sync struct {
|
|
Projects []*project `json:"projects"`
|
|
Items []*item `json:"items"`
|
|
Labels []*label `json:"labels"`
|
|
Notes []*note `json:"notes"`
|
|
ProjectNotes []*projectNote `json:"project_notes"`
|
|
Reminders []*reminder `json:"reminders"`
|
|
}
|
|
|
|
var todoistColors = map[int]string{}
|
|
|
|
func init() {
|
|
todoistColors = make(map[int]string, 19)
|
|
// The todoists colors are static, taken from https://developer.todoist.com/sync/v8/#colors
|
|
todoistColors = map[int]string{
|
|
30: "b8256f",
|
|
31: "db4035",
|
|
32: "ff9933",
|
|
33: "fad000",
|
|
34: "afb83b",
|
|
35: "7ecc49",
|
|
36: "299438",
|
|
37: "6accbc",
|
|
38: "158fad",
|
|
39: "14aaf5",
|
|
40: "96c3eb",
|
|
41: "4073ff",
|
|
42: "884dff",
|
|
43: "af38eb",
|
|
44: "eb96eb",
|
|
45: "e05194",
|
|
46: "ff8d85",
|
|
47: "808080",
|
|
48: "b8b8b8",
|
|
49: "ccac93",
|
|
}
|
|
}
|
|
|
|
// Name is used to get the name of the todoist migration - we're using the docs here to annotate the status route.
|
|
// @Summary Get migration status
|
|
// @Description Returns if the current user already did the migation or not. This is useful to show a confirmation message in the frontend if the user is trying to do the same migration again.
|
|
// @tags migration
|
|
// @Produce json
|
|
// @Security JWTKeyAuth
|
|
// @Success 200 {object} migration.Status "The migration status"
|
|
// @Failure 500 {object} models.Message "Internal server error"
|
|
// @Router /migration/todoist/status [get]
|
|
func (m *Migration) Name() string {
|
|
return "todoist"
|
|
}
|
|
|
|
// AuthURL returns the url users need to authenticate against
|
|
// @Summary Get the auth url from todoist
|
|
// @Description Returns the auth url where the user needs to get its auth code. This code can then be used to migrate everything from todoist to Vikunja.
|
|
// @tags migration
|
|
// @Produce json
|
|
// @Security JWTKeyAuth
|
|
// @Success 200 {object} handler.AuthURL "The auth url."
|
|
// @Failure 500 {object} models.Message "Internal server error"
|
|
// @Router /migration/todoist/auth [get]
|
|
func (m *Migration) AuthURL() string {
|
|
return "https://todoist.com/oauth/authorize" +
|
|
"?client_id=" + config.MigrationTodoistClientID.GetString() +
|
|
"&scope=data:read" +
|
|
"&state=" + utils.MakeRandomString(32)
|
|
}
|
|
|
|
func doPost(url string, form url.Values) (resp *http.Response, err error) {
|
|
req, err := http.NewRequest("POST", url, strings.NewReader(form.Encode()))
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
|
|
hc := http.Client{}
|
|
return hc.Do(req)
|
|
}
|
|
|
|
func convertTodoistToVikunja(sync *sync) (fullVikunjaHierachie []*models.NamespaceWithLists, err error) {
|
|
|
|
newNamespace := &models.NamespaceWithLists{
|
|
Namespace: models.Namespace{
|
|
Title: "Migrated from todoist",
|
|
},
|
|
}
|
|
|
|
// A map for all vikunja lists with the project id they're coming from as key
|
|
lists := make(map[int]*models.List, len(sync.Projects))
|
|
|
|
// A map for all vikunja tasks with the todoist task id as key to find them easily and add more data
|
|
tasks := make(map[int]*models.Task, len(sync.Items))
|
|
|
|
// A map for all vikunja labels with the todoist id as key to find them easier
|
|
labels := make(map[int]*models.Label, len(sync.Labels))
|
|
|
|
for _, p := range sync.Projects {
|
|
list := &models.List{
|
|
Title: p.Name,
|
|
HexColor: todoistColors[p.Color],
|
|
IsArchived: p.IsArchived == 1,
|
|
}
|
|
|
|
lists[p.ID] = list
|
|
|
|
newNamespace.Lists = append(newNamespace.Lists, list)
|
|
}
|
|
|
|
for _, label := range sync.Labels {
|
|
labels[label.ID] = &models.Label{
|
|
Title: label.Name,
|
|
HexColor: todoistColors[label.Color],
|
|
}
|
|
}
|
|
|
|
for _, i := range sync.Items {
|
|
task := &models.Task{
|
|
Title: i.Content,
|
|
Created: i.DateAdded.In(config.GetTimeZone()),
|
|
Done: i.Checked == 1,
|
|
}
|
|
|
|
// Only try to parse the task done at date if the task is actually done
|
|
// Sometimes weired things happen if we try to parse nil dates.
|
|
if task.Done {
|
|
task.DoneAt = i.DateCompleted.In(config.GetTimeZone())
|
|
}
|
|
|
|
// Todoist priorities only range from 1 (lowest) and max 4 (highest), so we need to make slight adjustments
|
|
if i.Priority > 1 {
|
|
task.Priority = int64(i.Priority)
|
|
}
|
|
|
|
// Put the due date together
|
|
if i.Due != nil {
|
|
dueDate, err := time.Parse("2006-01-02", i.Due.Date)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
task.DueDate = dueDate.In(config.GetTimeZone())
|
|
}
|
|
|
|
// Put all labels together from earlier
|
|
for _, lID := range i.Labels {
|
|
task.Labels = append(task.Labels, labels[lID])
|
|
}
|
|
|
|
tasks[i.ID] = task
|
|
|
|
lists[i.ProjectID].Tasks = append(lists[i.ProjectID].Tasks, task)
|
|
}
|
|
|
|
// If the parenId of a task is not 0, create a task relation
|
|
// We're looping again here to make sure we have seem all tasks before and have them in our map
|
|
for _, i := range sync.Items {
|
|
if i.ParentID == 0 {
|
|
continue
|
|
}
|
|
|
|
// Prevent all those nil errors
|
|
if tasks[i.ParentID].RelatedTasks == nil {
|
|
tasks[i.ParentID].RelatedTasks = make(models.RelatedTaskMap)
|
|
}
|
|
|
|
tasks[i.ParentID].RelatedTasks[models.RelationKindSubtask] = append(tasks[i.ParentID].RelatedTasks[models.RelationKindSubtask], tasks[i.ID])
|
|
}
|
|
|
|
// Task Notes -> Task Descriptions
|
|
for _, n := range sync.Notes {
|
|
if tasks[n.ItemID].Description != "" {
|
|
tasks[n.ItemID].Description += "\n"
|
|
}
|
|
tasks[n.ItemID].Description += n.Content
|
|
|
|
if n.FileAttachment == nil {
|
|
continue
|
|
}
|
|
|
|
// Download the attachment and put it in the file
|
|
resp, err := http.Get(n.FileAttachment.FileURL)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
buf := &bytes.Buffer{}
|
|
_, err = buf.ReadFrom(resp.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
tasks[n.ItemID].Attachments = append(tasks[n.ItemID].Attachments, &models.TaskAttachment{
|
|
File: &files.File{
|
|
Name: n.FileAttachment.FileName,
|
|
Mime: n.FileAttachment.FileType,
|
|
Size: uint64(n.FileAttachment.FileSize),
|
|
Created: n.Posted,
|
|
// We directly pass the file contents here to have a way to link the attachment to the file later.
|
|
// Because we don't have an ID for our task at this point of the migration, we cannot just throw all
|
|
// attachments in a slice and do the work of downloading and properly storing them later.
|
|
FileContent: buf.Bytes(),
|
|
},
|
|
Created: n.Posted,
|
|
})
|
|
}
|
|
|
|
// Project Notes -> List Descriptions
|
|
for _, pn := range sync.ProjectNotes {
|
|
if lists[pn.ProjectID].Description != "" {
|
|
lists[pn.ProjectID].Description += "\n"
|
|
}
|
|
|
|
lists[pn.ProjectID].Description += pn.Content
|
|
}
|
|
|
|
// Reminders -> vikunja reminders
|
|
for _, r := range sync.Reminders {
|
|
if r.Due == nil {
|
|
continue
|
|
}
|
|
|
|
date, err := time.Parse("2006-01-02", r.Due.Date)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
tasks[r.ItemID].Reminders = append(tasks[r.ItemID].Reminders, date.In(config.GetTimeZone()))
|
|
}
|
|
|
|
return []*models.NamespaceWithLists{
|
|
newNamespace,
|
|
}, err
|
|
}
|
|
|
|
func getAccessTokenFromAuthToken(authToken string) (accessToken string, err error) {
|
|
|
|
form := url.Values{
|
|
"client_id": []string{config.MigrationTodoistClientID.GetString()},
|
|
"client_secret": []string{config.MigrationTodoistClientSecret.GetString()},
|
|
"code": []string{authToken},
|
|
"redirect_uri": []string{config.MigrationTodoistRedirectURL.GetString()},
|
|
}
|
|
resp, err := doPost("https://todoist.com/oauth/access_token", form)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
if resp.StatusCode > 399 {
|
|
buf := &bytes.Buffer{}
|
|
_, _ = buf.ReadFrom(resp.Body)
|
|
return "", fmt.Errorf("got http status %d while trying to get token, error was %s", resp.StatusCode, buf.String())
|
|
}
|
|
|
|
token := &apiTokenResponse{}
|
|
err = json.NewDecoder(resp.Body).Decode(token)
|
|
return token.AccessToken, err
|
|
}
|
|
|
|
// Migrate gets all tasks from todoist for a user and puts them into vikunja
|
|
// @Summary Migrate all lists, tasks etc. from todoist
|
|
// @Description Migrates all projects, tasks, notes, reminders, subtasks and files from todoist to vikunja.
|
|
// @tags migration
|
|
// @Accept json
|
|
// @Produce json
|
|
// @Security JWTKeyAuth
|
|
// @Param migrationCode body todoist.Migration true "The auth code previously obtained from the auth url. See the docs for /migration/todoist/auth."
|
|
// @Success 200 {object} models.Message "A message telling you everything was migrated successfully."
|
|
// @Failure 500 {object} models.Message "Internal server error"
|
|
// @Router /migration/todoist/migrate [post]
|
|
func (m *Migration) Migrate(u *user.User) (err error) {
|
|
|
|
log.Debugf("[Todoist Migration] Starting migration for user %d", u.ID)
|
|
|
|
// 0. Get an api token from the obtained auth token
|
|
token, err := getAccessTokenFromAuthToken(m.Code)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
if token == "" {
|
|
log.Debugf("[Todoist Migration] Could not get token")
|
|
return
|
|
}
|
|
|
|
log.Debugf("[Todoist Migration] Got user token for user %d", u.ID)
|
|
log.Debugf("[Todoist Migration] Getting todoist data for user %d", u.ID)
|
|
|
|
// Get everything with the sync api
|
|
form := url.Values{
|
|
"token": []string{token},
|
|
"sync_token": []string{"*"},
|
|
"resource_types": []string{"[\"all\"]"},
|
|
}
|
|
resp, err := doPost("https://api.todoist.com/sync/v8/sync", form)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
syncResponse := &sync{}
|
|
err = json.NewDecoder(resp.Body).Decode(syncResponse)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
log.Debugf("[Todoist Migration] Got all todoist user data for user %d", u.ID)
|
|
log.Debugf("[Todoist Migration] Start converting data for user %d", u.ID)
|
|
|
|
fullVikunjaHierachie, err := convertTodoistToVikunja(syncResponse)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
log.Debugf("[Todoist Migration] Done converting data for user %d", u.ID)
|
|
log.Debugf("[Todoist Migration] Start inserting data for user %d", u.ID)
|
|
|
|
err = migration.InsertFromStructure(fullVikunjaHierachie, u)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
log.Debugf("[Todoist Migration] Done inserting data for user %d", u.ID)
|
|
log.Debugf("[Todoist Migration] Todoist migration done for user %d", u.ID)
|
|
|
|
return nil
|
|
}
|