fix(migration): import TickTick data by column name instead of index (#1356)

Resolves: https://github.com/go-vikunja/api/issues/61
Co-authored-by: kolaente <k@knt.li>
Reviewed-on: https://kolaente.dev/vikunja/api/pulls/1356
Co-authored-by: kooshi <kolaente.dev@pat.de.com>
Co-committed-by: kooshi <kolaente.dev@pat.de.com>

go.mod

@@ -98,6 +98,7 @@ require (
 	github.com/go-openapi/jsonreference v0.19.6 // indirect
 	github.com/go-openapi/spec v0.20.4 // indirect
 	github.com/go-openapi/swag v0.19.15 // indirect
+	github.com/gocarina/gocsv v0.0.0-20221216233619-1fea7ae8d380 // indirect
 	github.com/goccy/go-json v0.9.11 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/golang/protobuf v1.5.2 // indirect

go.sum

@@ -251,6 +251,8 @@ github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-testfixtures/testfixtures/v3 v3.8.1 h1:uonwvepqRvSgddcrReZQhojTlWlmOlHkYAb9ZaOMWgU=
 github.com/go-testfixtures/testfixtures/v3 v3.8.1/go.mod h1:Kdu7YeMC0KRXVHdaQ91Vmx3pcjoTF63h4f1qTJDdXLA=
+github.com/gocarina/gocsv v0.0.0-20221216233619-1fea7ae8d380 h1:JJq8YZiS07gFIMYZxkbbiMrXIglG3k5JPPtdvckcnfQ=
+github.com/gocarina/gocsv v0.0.0-20221216233619-1fea7ae8d380/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI=
 github.com/goccy/go-json v0.8.1/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk=
 github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=

pkg/modules/migration/ticktick/ticktick.go

@@ -27,10 +27,11 @@ import (
 	"time"
 
 	"code.vikunja.io/api/pkg/log"
-
 	"code.vikunja.io/api/pkg/models"
 	"code.vikunja.io/api/pkg/modules/migration"
 	"code.vikunja.io/api/pkg/user"
+
+	"github.com/gocarina/gocsv"
 )
 
 const timeISO = "2006-01-02T15:04:05-0700"
@@ -39,23 +40,39 @@ type Migrator struct {
 }
 
 type tickTickTask struct {
-	FolderName    string
+	FolderName        string        `csv:"Folder Name"`
-	ListName      string
+	ListName          string        `csv:"List Name"`
-	Title         string
+	Title             string        `csv:"Title"`
-	Tags          []string
+	TagsList          string        `csv:"Tags"`
-	Content       string
+	Tags              []string      `csv:"-"`
-	IsChecklist   bool
+	Content           string        `csv:"Content"`
-	StartDate     time.Time
+	IsChecklistString string        `csv:"Is Check list"`
-	DueDate       time.Time
+	IsChecklist       bool          `csv:"-"`
-	Reminder      time.Duration
+	StartDate         tickTickTime  `csv:"Start Date"`
-	Repeat        string
+	DueDate           tickTickTime  `csv:"Due Date"`
-	Priority      int
+	ReminderDuration  string        `csv:"Reminder"`
-	Status        string
+	Reminder          time.Duration `csv:"-"`
-	CreatedTime   time.Time
+	Repeat            string        `csv:"Repeat"`
-	CompletedTime time.Time
+	Priority          int           `csv:"Priority"`
-	Order         float64
+	Status            string        `csv:"Status"`
-	TaskID        int64
+	CreatedTime       tickTickTime  `csv:"Created Time"`
-	ParentID      int64
+	CompletedTime     tickTickTime  `csv:"Completed Time"`
+	Order             float64       `csv:"Order"`
+	TaskID            int64         `csv:"taskId"`
+	ParentID          int64         `csv:"parentId"`
+}
+
+type tickTickTime struct {
+	time.Time
+}
+
+func (date *tickTickTime) UnmarshalCSV(csv string) (err error) {
+	date.Time = time.Time{}
+	if csv == "" {
+		return nil
+	}
+	date.Time, err = time.Parse(timeISO, csv)
+	return err
 }
 
 // Copied from https://stackoverflow.com/a/57617885
@@ -119,19 +136,22 @@ func convertTickTickToVikunja(tasks []*tickTickTask) (result []*models.Namespace
 				ID:          t.TaskID,
 				Title:       t.Title,
 				Description: t.Content,
-				StartDate:   t.StartDate,
+				StartDate:   t.StartDate.Time,
-				EndDate:     t.DueDate,
+				EndDate:     t.DueDate.Time,
-				DueDate:     t.DueDate,
+				DueDate:     t.DueDate.Time,
-				Reminders: []time.Time{
+				Done:        t.Status == "1",
-					t.DueDate.Add(t.Reminder * -1),
+				DoneAt:      t.CompletedTime.Time,
-				},
+				Position:    t.Order,
-				Done:     t.Status == "1",
+				Labels:      labels,
-				DoneAt:   t.CompletedTime,
-				Position: t.Order,
-				Labels:   labels,
 			},
 		}
 
+		if !t.DueDate.IsZero() && t.Reminder > 0 {
+			task.Task.Reminders = []time.Time{
+				t.DueDate.Add(t.Reminder * -1),
+			}
+		}
+
 		if t.ParentID != 0 {
 			task.RelatedTasks = map[models.RelationKind][]*models.Task{
 				models.RelationKindParenttask: {{ID: t.ParentID}},
@@ -165,6 +185,22 @@ func (m *Migrator) Name() string {
 	return "ticktick"
 }
 
+func newLineSkipDecoder(r io.Reader, linesToSkip int) gocsv.SimpleDecoder {
+	reader := csv.NewReader(r)
+	//	reader.FieldsPerRecord = -1
+	for i := 0; i < linesToSkip; i++ {
+		_, err := reader.Read()
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			log.Debugf("[TickTick Migration] CSV parse error: %s", err)
+		}
+	}
+	reader.FieldsPerRecord = 0
+	return gocsv.NewSimpleDecoderFromCSVReader(reader)
+}
+
 // Migrate takes a ticktick export, parses it and imports everything in it into Vikunja.
 // @Summary Import all lists, tasks etc. from a TickTick backup export
 // @Description Imports all projects, tasks, notes, reminders, subtasks and files from a TickTick backup export into Vikunja.
@@ -178,85 +214,26 @@ func (m *Migrator) Name() string {
 // @Router /migration/ticktick/migrate [post]
 func (m *Migrator) Migrate(user *user.User, file io.ReaderAt, size int64) error {
 	fr := io.NewSectionReader(file, 0, size)
-	r := csv.NewReader(fr)
+	//r := csv.NewReader(fr)
 
 	allTasks := []*tickTickTask{}
-	line := 0
+	decode := newLineSkipDecoder(fr, 3)
-	for {
+	err := gocsv.UnmarshalDecoder(decode, &allTasks)
+	if err != nil {
+		return err
+	}
 
-		record, err := r.Read()
+	for _, task := range allTasks {
-		if err != nil {
+		if task.IsChecklistString == "Y" {
-			if errors.Is(err, io.EOF) {
+			task.IsChecklist = true
-				break
-			}
-			log.Debugf("[TickTick Migration] CSV parse error: %s", err)
 		}
 
-		line++
+		reminder := parseDuration(task.ReminderDuration)
-		if line <= 4 {
+		if reminder > 0 {
-			continue
+			task.Reminder = reminder
 		}
 
-		priority, err := strconv.Atoi(record[10])
+		task.Tags = strings.Split(task.TagsList, ", ")
-		if err != nil {
-			return err
-		}
-		order, err := strconv.ParseFloat(record[14], 64)
-		if err != nil {
-			return err
-		}
-		taskID, err := strconv.ParseInt(record[21], 10, 64)
-		if err != nil {
-			return err
-		}
-		parentID, err := strconv.ParseInt(record[21], 10, 64)
-		if err != nil {
-			return err
-		}
-
-		reminder := parseDuration(record[8])
-
-		t := &tickTickTask{
-			ListName:    record[1],
-			Title:       record[2],
-			Tags:        strings.Split(record[3], ", "),
-			Content:     record[4],
-			IsChecklist: record[5] == "Y",
-			Reminder:    reminder,
-			Repeat:      record[9],
-			Priority:    priority,
-			Status:      record[11],
-			Order:       order,
-			TaskID:      taskID,
-			ParentID:    parentID,
-		}
-
-		if record[6] != "" {
-			t.StartDate, err = time.Parse(timeISO, record[6])
-			if err != nil {
-				return err
-			}
-		}
-		if record[7] != "" {
-			t.DueDate, err = time.Parse(timeISO, record[7])
-			if err != nil {
-				return err
-			}
-		}
-		if record[12] != "" {
-			t.StartDate, err = time.Parse(timeISO, record[12])
-			if err != nil {
-				return err
-			}
-		}
-		if record[13] != "" {
-			t.CompletedTime, err = time.Parse(timeISO, record[13])
-			if err != nil {
-				return err
-			}
-		}
-
-		allTasks = append(allTasks, t)
 	}
 
 	vikunjaTasks := convertTickTickToVikunja(allTasks)

pkg/modules/migration/ticktick/ticktick_test.go

@@ -26,12 +26,15 @@ import (
 )
 
 func TestConvertTicktickTasksToVikunja(t *testing.T) {
-	time1, err := time.Parse(time.RFC3339Nano, "2022-11-18T03:00:00.4770000Z")
+	t1, err := time.Parse(time.RFC3339Nano, "2022-11-18T03:00:00.4770000Z")
 	require.NoError(t, err)
-	time2, err := time.Parse(time.RFC3339Nano, "2022-12-18T03:00:00.4770000Z")
+	time1 := tickTickTime{Time: t1}
+	t2, err := time.Parse(time.RFC3339Nano, "2022-12-18T03:00:00.4770000Z")
 	require.NoError(t, err)
-	time3, err := time.Parse(time.RFC3339Nano, "2022-12-10T03:00:00.4770000Z")
+	time2 := tickTickTime{Time: t2}
+	t3, err := time.Parse(time.RFC3339Nano, "2022-12-10T03:00:00.4770000Z")
 	require.NoError(t, err)
+	time3 := tickTickTime{Time: t3}
 	duration, err := time.ParseDuration("24h")
 	require.NoError(t, err)
 
@@ -91,9 +94,9 @@ func TestConvertTicktickTasksToVikunja(t *testing.T) {
 
 	assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Title, tickTickTasks[0].Title)
 	assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Description, tickTickTasks[0].Content)
-	assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].StartDate, tickTickTasks[0].StartDate)
+	assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].StartDate, tickTickTasks[0].StartDate.Time)
-	assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].EndDate, tickTickTasks[0].DueDate)
+	assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].EndDate, tickTickTasks[0].DueDate.Time)
-	assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].DueDate, tickTickTasks[0].DueDate)
+	assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].DueDate, tickTickTasks[0].DueDate.Time)
 	assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Labels, []*models.Label{
 		{Title: "label1"},
 		{Title: "label2"},
@@ -105,7 +108,7 @@ func TestConvertTicktickTasksToVikunja(t *testing.T) {
 	assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].Title, tickTickTasks[1].Title)
 	assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].Position, tickTickTasks[1].Order)
 	assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].Done, true)
-	assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].DoneAt, tickTickTasks[1].CompletedTime)
+	assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].DoneAt, tickTickTasks[1].CompletedTime.Time)
 	assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].RelatedTasks, models.RelatedTaskMap{
 		models.RelationKindParenttask: []*models.Task{
 			{
@@ -116,9 +119,9 @@ func TestConvertTicktickTasksToVikunja(t *testing.T) {
 
 	assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Title, tickTickTasks[2].Title)
 	assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Description, tickTickTasks[2].Content)
-	assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].StartDate, tickTickTasks[2].StartDate)
+	assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].StartDate, tickTickTasks[2].StartDate.Time)
-	assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].EndDate, tickTickTasks[2].DueDate)
+	assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].EndDate, tickTickTasks[2].DueDate.Time)
-	assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].DueDate, tickTickTasks[2].DueDate)
+	assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].DueDate, tickTickTasks[2].DueDate.Time)
 	assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Labels, []*models.Label{
 		{Title: "label1"},
 		{Title: "label2"},