ami/tl-vikunja
1
0
Fork 0
Code

fix(migration): import TickTick data by column name instead of index (#1356)

Browse Source 
Resolves: https://github.com/go-vikunja/api/issues/61
Co-authored-by: kolaente <k@knt.li>
Reviewed-on: https://kolaente.dev/vikunja/api/pulls/1356
Co-authored-by: kooshi <kolaente.dev@pat.de.com>
Co-committed-by: kooshi <kolaente.dev@pat.de.com>
This commit is contained in:
kooshi 2023-01-24 13:58:18 +00:00 committed by konrad
parent 530bb0a63c
commit 31a1452839
4 changed files with 93 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod
View File

@ -98,6 +98,7 @@ require (
github.com/go-openapi/jsonreference v0.19.6 // indirect
github.com/go-openapi/spec v0.20.4 // indirect
github.com/go-openapi/swag v0.19.15 // indirect
github.com/gocarina/gocsv v0.0.0-20221216233619-1fea7ae8d380 // indirect
github.com/goccy/go-json v0.9.11 // indirect
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
github.com/golang/protobuf v1.5.2 // indirect

2
go.sum
View File

@ -251,6 +251,8 @@ github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-testfixtures/testfixtures/v3 v3.8.1 h1:uonwvepqRvSgddcrReZQhojTlWlmOlHkYAb9ZaOMWgU=
github.com/go-testfixtures/testfixtures/v3 v3.8.1/go.mod h1:Kdu7YeMC0KRXVHdaQ91Vmx3pcjoTF63h4f1qTJDdXLA=
github.com/gocarina/gocsv v0.0.0-20221216233619-1fea7ae8d380 h1:JJq8YZiS07gFIMYZxkbbiMrXIglG3k5JPPtdvckcnfQ=
github.com/gocarina/gocsv v0.0.0-20221216233619-1fea7ae8d380/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI=
github.com/goccy/go-json v0.8.1/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk=
github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=

165
pkg/modules/migration/ticktick/ticktick.go
View File

@ -27,10 +27,11 @@ import (
"time"
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/models"
"code.vikunja.io/api/pkg/modules/migration"
"code.vikunja.io/api/pkg/user"
"github.com/gocarina/gocsv"
)
const timeISO = "2006-01-02T15:04:05-0700"
@ -39,23 +40,39 @@ type Migrator struct {
}
type tickTickTask struct {
FolderName string
ListName string
Title string
Tags []string
Content string
IsChecklist bool
StartDate time.Time
DueDate time.Time
Reminder time.Duration
Repeat string
Priority int
Status string
CreatedTime time.Time
CompletedTime time.Time
Order float64
TaskID int64
ParentID int64
FolderName string `csv:"Folder Name"`
ListName string `csv:"List Name"`
Title string `csv:"Title"`
TagsList string `csv:"Tags"`
Tags []string `csv:"-"`
Content string `csv:"Content"`
IsChecklistString string `csv:"Is Check list"`
IsChecklist bool `csv:"-"`
StartDate tickTickTime `csv:"Start Date"`
DueDate tickTickTime `csv:"Due Date"`
ReminderDuration string `csv:"Reminder"`
Reminder time.Duration `csv:"-"`
Repeat string `csv:"Repeat"`
Priority int `csv:"Priority"`
Status string `csv:"Status"`
CreatedTime tickTickTime `csv:"Created Time"`
CompletedTime tickTickTime `csv:"Completed Time"`
Order float64 `csv:"Order"`
TaskID int64 `csv:"taskId"`
ParentID int64 `csv:"parentId"`
}
type tickTickTime struct {
time.Time
}
func (date *tickTickTime) UnmarshalCSV(csv string) (err error) {
date.Time = time.Time{}
if csv == "" {
return nil
}
date.Time, err = time.Parse(timeISO, csv)
return err
}
// Copied from https://stackoverflow.com/a/57617885
@ -119,19 +136,22 @@ func convertTickTickToVikunja(tasks []*tickTickTask) (result []*models.Namespace
ID: t.TaskID,
Title: t.Title,
Description: t.Content,
StartDate: t.StartDate,
EndDate: t.DueDate,
DueDate: t.DueDate,
Reminders: []time.Time{
t.DueDate.Add(t.Reminder * -1),
},
StartDate: t.StartDate.Time,
EndDate: t.DueDate.Time,
DueDate: t.DueDate.Time,
Done: t.Status == "1",
DoneAt: t.CompletedTime,
DoneAt: t.CompletedTime.Time,
Position: t.Order,
Labels: labels,
},
}
if !t.DueDate.IsZero() && t.Reminder > 0 {
task.Task.Reminders = []time.Time{
t.DueDate.Add(t.Reminder * -1),
}
}
if t.ParentID != 0 {
task.RelatedTasks = map[models.RelationKind][]*models.Task{
models.RelationKindParenttask: {{ID: t.ParentID}},
@ -165,6 +185,22 @@ func (m *Migrator) Name() string {
return "ticktick"
}
func newLineSkipDecoder(r io.Reader, linesToSkip int) gocsv.SimpleDecoder {
reader := csv.NewReader(r)
// reader.FieldsPerRecord = -1
for i := 0; i < linesToSkip; i++ {
_, err := reader.Read()
if err != nil {
if errors.Is(err, io.EOF) {
break
}
log.Debugf("[TickTick Migration] CSV parse error: %s", err)
}
}
reader.FieldsPerRecord = 0
return gocsv.NewSimpleDecoderFromCSVReader(reader)
}
// Migrate takes a ticktick export, parses it and imports everything in it into Vikunja.
// @Summary Import all lists, tasks etc. from a TickTick backup export
// @Description Imports all projects, tasks, notes, reminders, subtasks and files from a TickTick backup export into Vikunja.
@ -178,85 +214,26 @@ func (m *Migrator) Name() string {
// @Router /migration/ticktick/migrate [post]
func (m *Migrator) Migrate(user *user.User, file io.ReaderAt, size int64) error {
fr := io.NewSectionReader(file, 0, size)
r := csv.NewReader(fr)
//r := csv.NewReader(fr)
allTasks := []*tickTickTask{}
line := 0
for {
record, err := r.Read()
if err != nil {
if errors.Is(err, io.EOF) {
break
}
log.Debugf("[TickTick Migration] CSV parse error: %s", err)
}
line++
if line <= 4 {
continue
}
priority, err := strconv.Atoi(record[10])
if err != nil {
return err
}
order, err := strconv.ParseFloat(record[14], 64)
if err != nil {
return err
}
taskID, err := strconv.ParseInt(record[21], 10, 64)
if err != nil {
return err
}
parentID, err := strconv.ParseInt(record[21], 10, 64)
decode := newLineSkipDecoder(fr, 3)
err := gocsv.UnmarshalDecoder(decode, &allTasks)
if err != nil {
return err
}
reminder := parseDuration(record[8])
t := &tickTickTask{
ListName: record[1],
Title: record[2],
Tags: strings.Split(record[3], ", "),
Content: record[4],
IsChecklist: record[5] == "Y",
Reminder: reminder,
Repeat: record[9],
Priority: priority,
Status: record[11],
Order: order,
TaskID: taskID,
ParentID: parentID,
for _, task := range allTasks {
if task.IsChecklistString == "Y" {
task.IsChecklist = true
}
if record[6] != "" {
t.StartDate, err = time.Parse(timeISO, record[6])
if err != nil {
return err
}
}
if record[7] != "" {
t.DueDate, err = time.Parse(timeISO, record[7])
if err != nil {
return err
}
}
if record[12] != "" {
t.StartDate, err = time.Parse(timeISO, record[12])
if err != nil {
return err
}
}
if record[13] != "" {
t.CompletedTime, err = time.Parse(timeISO, record[13])
if err != nil {
return err
}
reminder := parseDuration(task.ReminderDuration)
if reminder > 0 {
task.Reminder = reminder
}
allTasks = append(allTasks, t)
task.Tags = strings.Split(task.TagsList, ", ")
}
vikunjaTasks := convertTickTickToVikunja(allTasks)

23
pkg/modules/migration/ticktick/ticktick_test.go
View File

@ -26,12 +26,15 @@ import (
)
func TestConvertTicktickTasksToVikunja(t *testing.T) {
time1, err := time.Parse(time.RFC3339Nano, "2022-11-18T03:00:00.4770000Z")
t1, err := time.Parse(time.RFC3339Nano, "2022-11-18T03:00:00.4770000Z")
require.NoError(t, err)
time2, err := time.Parse(time.RFC3339Nano, "2022-12-18T03:00:00.4770000Z")
time1 := tickTickTime{Time: t1}
t2, err := time.Parse(time.RFC3339Nano, "2022-12-18T03:00:00.4770000Z")
require.NoError(t, err)
time3, err := time.Parse(time.RFC3339Nano, "2022-12-10T03:00:00.4770000Z")
time2 := tickTickTime{Time: t2}
t3, err := time.Parse(time.RFC3339Nano, "2022-12-10T03:00:00.4770000Z")
require.NoError(t, err)
time3 := tickTickTime{Time: t3}
duration, err := time.ParseDuration("24h")
require.NoError(t, err)
@ -91,9 +94,9 @@ func TestConvertTicktickTasksToVikunja(t *testing.T) {
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Title, tickTickTasks[0].Title)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Description, tickTickTasks[0].Content)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].StartDate, tickTickTasks[0].StartDate)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].EndDate, tickTickTasks[0].DueDate)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].DueDate, tickTickTasks[0].DueDate)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].StartDate, tickTickTasks[0].StartDate.Time)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].EndDate, tickTickTasks[0].DueDate.Time)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].DueDate, tickTickTasks[0].DueDate.Time)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Labels, []*models.Label{
{Title: "label1"},
{Title: "label2"},
@ -105,7 +108,7 @@ func TestConvertTicktickTasksToVikunja(t *testing.T) {
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].Title, tickTickTasks[1].Title)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].Position, tickTickTasks[1].Order)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].Done, true)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].DoneAt, tickTickTasks[1].CompletedTime)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].DoneAt, tickTickTasks[1].CompletedTime.Time)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].RelatedTasks, models.RelatedTaskMap{
models.RelationKindParenttask: []*models.Task{
{
@ -116,9 +119,9 @@ func TestConvertTicktickTasksToVikunja(t *testing.T) {
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Title, tickTickTasks[2].Title)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Description, tickTickTasks[2].Content)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].StartDate, tickTickTasks[2].StartDate)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].EndDate, tickTickTasks[2].DueDate)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].DueDate, tickTickTasks[2].DueDate)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].StartDate, tickTickTasks[2].StartDate.Time)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].EndDate, tickTickTasks[2].DueDate.Time)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].DueDate, tickTickTasks[2].DueDate.Time)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Labels, []*models.Label{
{Title: "label1"},
{Title: "label2"},