feat: TickTick migrator (#1273)

Reviewed-on: https://kolaente.dev/vikunja/api/pulls/1273
This commit is contained in:
konrad 2022-10-09 21:12:30 +00:00
commit df2e36c2a3
7 changed files with 617 additions and 12 deletions

View file

@ -0,0 +1,265 @@
// Vikunja is a to-do list application to facilitate your life.
// Copyright 2018-2021 Vikunja and contributors. All rights reserved.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public Licensee as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public Licensee for more details.
//
// You should have received a copy of the GNU Affero General Public Licensee
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package ticktick
import (
"encoding/csv"
"errors"
"io"
"regexp"
"sort"
"strconv"
"strings"
"time"
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/models"
"code.vikunja.io/api/pkg/modules/migration"
"code.vikunja.io/api/pkg/user"
)
const timeISO = "2006-01-02T15:04:05-0700"
type Migrator struct {
}
type tickTickTask struct {
FolderName string
ListName string
Title string
Tags []string
Content string
IsChecklist bool
StartDate time.Time
DueDate time.Time
Reminder time.Duration
Repeat string
Priority int
Status string
CreatedTime time.Time
CompletedTime time.Time
Order float64
TaskID int64
ParentID int64
}
// Copied from https://stackoverflow.com/a/57617885
var durationRegex = regexp.MustCompile(`P([\d\.]+Y)?([\d\.]+M)?([\d\.]+D)?T?([\d\.]+H)?([\d\.]+M)?([\d\.]+?S)?`)
// ParseDuration converts a ISO8601 duration into a time.Duration
func parseDuration(str string) time.Duration {
matches := durationRegex.FindStringSubmatch(str)
if len(matches) == 0 {
return 0
}
years := parseDurationPart(matches[1], time.Hour*24*365)
months := parseDurationPart(matches[2], time.Hour*24*30)
days := parseDurationPart(matches[3], time.Hour*24)
hours := parseDurationPart(matches[4], time.Hour)
minutes := parseDurationPart(matches[5], time.Second*60)
seconds := parseDurationPart(matches[6], time.Second)
return years + months + days + hours + minutes + seconds
}
func parseDurationPart(value string, unit time.Duration) time.Duration {
if len(value) != 0 {
if parsed, err := strconv.ParseFloat(value[:len(value)-1], 64); err == nil {
return time.Duration(float64(unit) * parsed)
}
}
return 0
}
func convertTickTickToVikunja(tasks []*tickTickTask) (result []*models.NamespaceWithListsAndTasks) {
namespace := &models.NamespaceWithListsAndTasks{
Namespace: models.Namespace{
Title: "Migrated from TickTick",
},
Lists: []*models.ListWithTasksAndBuckets{},
}
lists := make(map[string]*models.ListWithTasksAndBuckets)
for _, t := range tasks {
_, has := lists[t.ListName]
if !has {
lists[t.ListName] = &models.ListWithTasksAndBuckets{
List: models.List{
Title: t.ListName,
},
}
}
labels := make([]*models.Label, 0, len(t.Tags))
for _, tag := range t.Tags {
labels = append(labels, &models.Label{
Title: tag,
})
}
task := &models.TaskWithComments{
Task: models.Task{
ID: t.TaskID,
Title: t.Title,
Description: t.Content,
StartDate: t.StartDate,
EndDate: t.DueDate,
DueDate: t.DueDate,
Reminders: []time.Time{
t.DueDate.Add(t.Reminder * -1),
},
Done: t.Status == "1",
DoneAt: t.CompletedTime,
Position: t.Order,
Labels: labels,
},
}
if t.ParentID != 0 {
task.RelatedTasks = map[models.RelationKind][]*models.Task{
models.RelationKindParenttask: {{ID: t.ParentID}},
}
}
lists[t.ListName].Tasks = append(lists[t.ListName].Tasks, task)
}
for _, l := range lists {
namespace.Lists = append(namespace.Lists, l)
}
sort.Slice(namespace.Lists, func(i, j int) bool {
return namespace.Lists[i].Title < namespace.Lists[j].Title
})
return []*models.NamespaceWithListsAndTasks{namespace}
}
// Name is used to get the name of the ticktick migration - we're using the docs here to annotate the status route.
// @Summary Get migration status
// @Description Returns if the current user already did the migation or not. This is useful to show a confirmation message in the frontend if the user is trying to do the same migration again.
// @tags migration
// @Produce json
// @Security JWTKeyAuth
// @Success 200 {object} migration.Status "The migration status"
// @Failure 500 {object} models.Message "Internal server error"
// @Router /migration/ticktick/status [get]
func (m *Migrator) Name() string {
return "ticktick"
}
// Migrate takes a ticktick export, parses it and imports everything in it into Vikunja.
// @Summary Import all lists, tasks etc. from a TickTick backup export
// @Description Imports all projects, tasks, notes, reminders, subtasks and files from a TickTick backup export into Vikunja.
// @tags migration
// @Accept json
// @Produce json
// @Security JWTKeyAuth
// @Param import formData string true "The TickTick backup csv file."
// @Success 200 {object} models.Message "A message telling you everything was migrated successfully."
// @Failure 500 {object} models.Message "Internal server error"
// @Router /migration/ticktick/migrate [post]
func (m *Migrator) Migrate(user *user.User, file io.ReaderAt, size int64) error {
fr := io.NewSectionReader(file, 0, size)
r := csv.NewReader(fr)
allTasks := []*tickTickTask{}
line := 0
for {
record, err := r.Read()
if err != nil {
if errors.Is(err, io.EOF) {
break
}
log.Debugf("[TickTick Migration] CSV parse error: %s", err)
}
line++
if line <= 4 {
continue
}
priority, err := strconv.Atoi(record[10])
if err != nil {
return err
}
order, err := strconv.ParseFloat(record[14], 64)
if err != nil {
return err
}
taskID, err := strconv.ParseInt(record[21], 10, 64)
if err != nil {
return err
}
parentID, err := strconv.ParseInt(record[21], 10, 64)
if err != nil {
return err
}
reminder := parseDuration(record[8])
t := &tickTickTask{
ListName: record[1],
Title: record[2],
Tags: strings.Split(record[3], ", "),
Content: record[4],
IsChecklist: record[5] == "Y",
Reminder: reminder,
Repeat: record[9],
Priority: priority,
Status: record[11],
Order: order,
TaskID: taskID,
ParentID: parentID,
}
if record[6] != "" {
t.StartDate, err = time.Parse(timeISO, record[6])
if err != nil {
return err
}
}
if record[7] != "" {
t.DueDate, err = time.Parse(timeISO, record[7])
if err != nil {
return err
}
}
if record[12] != "" {
t.StartDate, err = time.Parse(timeISO, record[12])
if err != nil {
return err
}
}
if record[13] != "" {
t.CompletedTime, err = time.Parse(timeISO, record[13])
if err != nil {
return err
}
}
allTasks = append(allTasks, t)
}
vikunjaTasks := convertTickTickToVikunja(allTasks)
return migration.InsertFromStructure(vikunjaTasks, user)
}

View file

@ -0,0 +1,136 @@
// Vikunja is a to-do list application to facilitate your life.
// Copyright 2018-2021 Vikunja and contributors. All rights reserved.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public Licensee as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public Licensee for more details.
//
// You should have received a copy of the GNU Affero General Public Licensee
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package ticktick
import (
"testing"
"time"
"code.vikunja.io/api/pkg/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestConvertTicktickTasksToVikunja(t *testing.T) {
time1, err := time.Parse(time.RFC3339Nano, "2022-11-18T03:00:00.4770000Z")
require.NoError(t, err)
time2, err := time.Parse(time.RFC3339Nano, "2022-12-18T03:00:00.4770000Z")
require.NoError(t, err)
time3, err := time.Parse(time.RFC3339Nano, "2022-12-10T03:00:00.4770000Z")
require.NoError(t, err)
duration, err := time.ParseDuration("24h")
require.NoError(t, err)
tickTickTasks := []*tickTickTask{
{
TaskID: 1,
ParentID: 0,
ListName: "List 1",
Title: "Test task 1",
Tags: []string{"label1", "label2"},
Content: "Lorem Ipsum Dolor sit amet",
StartDate: time1,
DueDate: time2,
Reminder: duration,
Repeat: "FREQ=WEEKLY;INTERVAL=1;UNTIL=20190117T210000Z",
Status: "0",
Order: -1099511627776,
},
{
TaskID: 2,
ParentID: 1,
ListName: "List 1",
Title: "Test task 2",
Status: "1",
CompletedTime: time3,
Order: -1099511626,
},
{
TaskID: 3,
ParentID: 0,
ListName: "List 1",
Title: "Test task 3",
Tags: []string{"label1", "label2", "other label"},
StartDate: time1,
DueDate: time2,
Reminder: duration,
Status: "0",
Order: -109951627776,
},
{
TaskID: 4,
ParentID: 0,
ListName: "List 2",
Title: "Test task 4",
Status: "0",
Order: -109951627777,
},
}
vikunjaTasks := convertTickTickToVikunja(tickTickTasks)
assert.Len(t, vikunjaTasks, 1)
assert.Len(t, vikunjaTasks[0].Lists, 2)
assert.Len(t, vikunjaTasks[0].Lists[0].Tasks, 3)
assert.Equal(t, vikunjaTasks[0].Lists[0].Title, tickTickTasks[0].ListName)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Title, tickTickTasks[0].Title)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Description, tickTickTasks[0].Content)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].StartDate, tickTickTasks[0].StartDate)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].EndDate, tickTickTasks[0].DueDate)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].DueDate, tickTickTasks[0].DueDate)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Labels, []*models.Label{
{Title: "label1"},
{Title: "label2"},
})
//assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Reminders, tickTickTasks[0].) // TODO
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Position, tickTickTasks[0].Order)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Done, false)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].Title, tickTickTasks[1].Title)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].Position, tickTickTasks[1].Order)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].Done, true)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].DoneAt, tickTickTasks[1].CompletedTime)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].RelatedTasks, models.RelatedTaskMap{
models.RelationKindParenttask: []*models.Task{
{
ID: tickTickTasks[1].ParentID,
},
},
})
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Title, tickTickTasks[2].Title)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Description, tickTickTasks[2].Content)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].StartDate, tickTickTasks[2].StartDate)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].EndDate, tickTickTasks[2].DueDate)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].DueDate, tickTickTasks[2].DueDate)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Labels, []*models.Label{
{Title: "label1"},
{Title: "label2"},
{Title: "other label"},
})
//assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Reminders, tickTickTasks[0].) // TODO
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Position, tickTickTasks[2].Order)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Done, false)
assert.Len(t, vikunjaTasks[0].Lists[1].Tasks, 1)
assert.Equal(t, vikunjaTasks[0].Lists[1].Title, tickTickTasks[3].ListName)
assert.Equal(t, vikunjaTasks[0].Lists[1].Tasks[0].Title, tickTickTasks[3].Title)
assert.Equal(t, vikunjaTasks[0].Lists[1].Tasks[0].Position, tickTickTasks[3].Order)
}

View file

@ -19,19 +19,17 @@ package v1
import ( import (
"net/http" "net/http"
vikunja_file "code.vikunja.io/api/pkg/modules/migration/vikunja-file"
microsofttodo "code.vikunja.io/api/pkg/modules/migration/microsoft-todo"
"code.vikunja.io/api/pkg/modules/migration/trello"
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/config" "code.vikunja.io/api/pkg/config"
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/modules/auth/openid" "code.vikunja.io/api/pkg/modules/auth/openid"
microsofttodo "code.vikunja.io/api/pkg/modules/migration/microsoft-todo"
"code.vikunja.io/api/pkg/modules/migration/ticktick"
"code.vikunja.io/api/pkg/modules/migration/todoist" "code.vikunja.io/api/pkg/modules/migration/todoist"
"code.vikunja.io/api/pkg/modules/migration/trello"
vikunja_file "code.vikunja.io/api/pkg/modules/migration/vikunja-file"
"code.vikunja.io/api/pkg/modules/migration/wunderlist" "code.vikunja.io/api/pkg/modules/migration/wunderlist"
"code.vikunja.io/api/pkg/version" "code.vikunja.io/api/pkg/version"
"github.com/labstack/echo/v4" "github.com/labstack/echo/v4"
) )
@ -97,6 +95,7 @@ func Info(c echo.Context) error {
TaskCommentsEnabled: config.ServiceEnableTaskComments.GetBool(), TaskCommentsEnabled: config.ServiceEnableTaskComments.GetBool(),
AvailableMigrators: []string{ AvailableMigrators: []string{
(&vikunja_file.FileMigrator{}).Name(), (&vikunja_file.FileMigrator{}).Name(),
(&ticktick.Migrator{}).Name(),
}, },
Legal: legalInfo{ Legal: legalInfo{
ImprintURL: config.LegalImprintURL.GetString(), ImprintURL: config.LegalImprintURL.GetString(),

View file

@ -53,10 +53,6 @@ import (
"strings" "strings"
"time" "time"
"github.com/ulule/limiter/v3"
vikunja_file "code.vikunja.io/api/pkg/modules/migration/vikunja-file"
"code.vikunja.io/api/pkg/config" "code.vikunja.io/api/pkg/config"
"code.vikunja.io/api/pkg/db" "code.vikunja.io/api/pkg/db"
"code.vikunja.io/api/pkg/log" "code.vikunja.io/api/pkg/log"
@ -70,8 +66,10 @@ import (
"code.vikunja.io/api/pkg/modules/migration" "code.vikunja.io/api/pkg/modules/migration"
migrationHandler "code.vikunja.io/api/pkg/modules/migration/handler" migrationHandler "code.vikunja.io/api/pkg/modules/migration/handler"
microsofttodo "code.vikunja.io/api/pkg/modules/migration/microsoft-todo" microsofttodo "code.vikunja.io/api/pkg/modules/migration/microsoft-todo"
"code.vikunja.io/api/pkg/modules/migration/ticktick"
"code.vikunja.io/api/pkg/modules/migration/todoist" "code.vikunja.io/api/pkg/modules/migration/todoist"
"code.vikunja.io/api/pkg/modules/migration/trello" "code.vikunja.io/api/pkg/modules/migration/trello"
vikunja_file "code.vikunja.io/api/pkg/modules/migration/vikunja-file"
"code.vikunja.io/api/pkg/modules/migration/wunderlist" "code.vikunja.io/api/pkg/modules/migration/wunderlist"
apiv1 "code.vikunja.io/api/pkg/routes/api/v1" apiv1 "code.vikunja.io/api/pkg/routes/api/v1"
"code.vikunja.io/api/pkg/routes/caldav" "code.vikunja.io/api/pkg/routes/caldav"
@ -86,6 +84,7 @@ import (
"github.com/labstack/echo/v4" "github.com/labstack/echo/v4"
"github.com/labstack/echo/v4/middleware" "github.com/labstack/echo/v4/middleware"
elog "github.com/labstack/gommon/log" elog "github.com/labstack/gommon/log"
"github.com/ulule/limiter/v3"
) )
// NewEcho registers a new Echo instance // NewEcho registers a new Echo instance
@ -653,12 +652,21 @@ func registerMigrations(m *echo.Group) {
microsoftTodoMigrationHandler.RegisterRoutes(m) microsoftTodoMigrationHandler.RegisterRoutes(m)
} }
// Vikunja File Migrator
vikunjaFileMigrationHandler := &migrationHandler.FileMigratorWeb{ vikunjaFileMigrationHandler := &migrationHandler.FileMigratorWeb{
MigrationStruct: func() migration.FileMigrator { MigrationStruct: func() migration.FileMigrator {
return &vikunja_file.FileMigrator{} return &vikunja_file.FileMigrator{}
}, },
} }
vikunjaFileMigrationHandler.RegisterRoutes(m) vikunjaFileMigrationHandler.RegisterRoutes(m)
// TickTick File Migrator
tickTickFileMigrator := migrationHandler.FileMigratorWeb{
MigrationStruct: func() migration.FileMigrator {
return &ticktick.Migrator{}
},
}
tickTickFileMigrator.RegisterRoutes(m)
} }
func registerCalDavRoutes(c *echo.Group) { func registerCalDavRoutes(c *echo.Group) {

View file

@ -2699,6 +2699,80 @@ const docTemplate = `{
} }
} }
}, },
"/migration/ticktick/migrate": {
"post": {
"security": [
{
"JWTKeyAuth": []
}
],
"description": "Imports all projects, tasks, notes, reminders, subtasks and files from a TickTick backup export into Vikunja.",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"migration"
],
"summary": "Import all lists, tasks etc. from a TickTick backup export",
"parameters": [
{
"type": "string",
"description": "The TickTick backup csv file.",
"name": "import",
"in": "formData",
"required": true
}
],
"responses": {
"200": {
"description": "A message telling you everything was migrated successfully.",
"schema": {
"$ref": "#/definitions/models.Message"
}
},
"500": {
"description": "Internal server error",
"schema": {
"$ref": "#/definitions/models.Message"
}
}
}
}
},
"/migration/ticktick/status": {
"get": {
"security": [
{
"JWTKeyAuth": []
}
],
"description": "Returns if the current user already did the migation or not. This is useful to show a confirmation message in the frontend if the user is trying to do the same migration again.",
"produces": [
"application/json"
],
"tags": [
"migration"
],
"summary": "Get migration status",
"responses": {
"200": {
"description": "The migration status",
"schema": {
"$ref": "#/definitions/migration.Status"
}
},
"500": {
"description": "Internal server error",
"schema": {
"$ref": "#/definitions/models.Message"
}
}
}
}
},
"/migration/todoist/auth": { "/migration/todoist/auth": {
"get": { "get": {
"security": [ "security": [

View file

@ -2690,6 +2690,80 @@
} }
} }
}, },
"/migration/ticktick/migrate": {
"post": {
"security": [
{
"JWTKeyAuth": []
}
],
"description": "Imports all projects, tasks, notes, reminders, subtasks and files from a TickTick backup export into Vikunja.",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"migration"
],
"summary": "Import all lists, tasks etc. from a TickTick backup export",
"parameters": [
{
"type": "string",
"description": "The TickTick backup csv file.",
"name": "import",
"in": "formData",
"required": true
}
],
"responses": {
"200": {
"description": "A message telling you everything was migrated successfully.",
"schema": {
"$ref": "#/definitions/models.Message"
}
},
"500": {
"description": "Internal server error",
"schema": {
"$ref": "#/definitions/models.Message"
}
}
}
}
},
"/migration/ticktick/status": {
"get": {
"security": [
{
"JWTKeyAuth": []
}
],
"description": "Returns if the current user already did the migation or not. This is useful to show a confirmation message in the frontend if the user is trying to do the same migration again.",
"produces": [
"application/json"
],
"tags": [
"migration"
],
"summary": "Get migration status",
"responses": {
"200": {
"description": "The migration status",
"schema": {
"$ref": "#/definitions/migration.Status"
}
},
"500": {
"description": "Internal server error",
"schema": {
"$ref": "#/definitions/models.Message"
}
}
}
}
},
"/migration/todoist/auth": { "/migration/todoist/auth": {
"get": { "get": {
"security": [ "security": [

View file

@ -3250,6 +3250,55 @@ paths:
summary: Get migration status summary: Get migration status
tags: tags:
- migration - migration
/migration/ticktick/migrate:
post:
consumes:
- application/json
description: Imports all projects, tasks, notes, reminders, subtasks and files
from a TickTick backup export into Vikunja.
parameters:
- description: The TickTick backup csv file.
in: formData
name: import
required: true
type: string
produces:
- application/json
responses:
"200":
description: A message telling you everything was migrated successfully.
schema:
$ref: '#/definitions/models.Message'
"500":
description: Internal server error
schema:
$ref: '#/definitions/models.Message'
security:
- JWTKeyAuth: []
summary: Import all lists, tasks etc. from a TickTick backup export
tags:
- migration
/migration/ticktick/status:
get:
description: Returns if the current user already did the migation or not. This
is useful to show a confirmation message in the frontend if the user is trying
to do the same migration again.
produces:
- application/json
responses:
"200":
description: The migration status
schema:
$ref: '#/definitions/migration.Status'
"500":
description: Internal server error
schema:
$ref: '#/definitions/models.Message'
security:
- JWTKeyAuth: []
summary: Get migration status
tags:
- migration
/migration/todoist/auth: /migration/todoist/auth:
get: get:
description: Returns the auth url where the user needs to get its auth code. description: Returns the auth url where the user needs to get its auth code.