feat(migration): Trello organization based migration (#2211)

Migrate Trello organization after organization to limit total memory allocation.
Related discussion: https://community.vikunja.io/t/trello-import-issues/2110

Co-authored-by: Elscrux <nickposer2102@gmail.com>
Co-authored-by: konrad <k@knt.li>
Co-authored-by: kolaente <k@knt.li>
Reviewed-on: https://kolaente.dev/vikunja/vikunja/pulls/2211
Reviewed-by: konrad <k@knt.li>
Co-authored-by: Elscrux <elscrux@gmail.com>
Co-committed-by: Elscrux <elscrux@gmail.com>

pkg/modules/migration/trello/trello.go

@@ -107,80 +107,100 @@ func (m *Migration) AuthURL() string {
 		"&return_url=" + config.MigrationTrelloRedirectURL.GetString()
 }
 
-func getTrelloData(token string) (trelloData []*trello.Board, err error) {
+func getTrelloBoards(client *trello.Client) (trelloData []*trello.Board, err error) {
-	allArg := trello.Arguments{"fields": "all"}
-
-	client := trello.NewClient(config.MigrationTrelloKey.GetString(), token)
-	client.Logger = log.GetLogger()
-
 	log.Debugf("[Trello Migration] Getting boards...")
 
 	trelloData, err = client.GetMyBoards(trello.Defaults())
 	if err != nil {
-		return
+		return nil, err
 	}
 
 	log.Debugf("[Trello Migration] Got %d trello boards", len(trelloData))
 
-	for _, board := range trelloData {
+	return
-		log.Debugf("[Trello Migration] Getting projects for board %s", board.ID)
+}
 
-		board.Lists, err = board.GetLists(trello.Defaults())
+func getTrelloOrganizationsWithBoards(boards []*trello.Board) (boardsByOrg map[string][]*trello.Board) {
+
+	boardsByOrg = make(map[string][]*trello.Board)
+
+	for _, board := range boards {
+		// Trello boards without an organization are considered personal boards
+		if board.IDOrganization == "" {
+			board.IDOrganization = "Personal"
+		}
+
+		_, has := boardsByOrg[board.IDOrganization]
+		if !has {
+			boardsByOrg[board.IDOrganization] = []*trello.Board{}
+		}
+
+		boardsByOrg[board.IDOrganization] = append(boardsByOrg[board.IDOrganization], board)
+	}
+
+	return
+}
+
+func fillCardData(client *trello.Client, board *trello.Board) (err error) {
+	allArg := trello.Arguments{"fields": "all"}
+
+	log.Debugf("[Trello Migration] Getting projects for board %s", board.ID)
+
+	board.Lists, err = board.GetLists(trello.Defaults())
+	if err != nil {
+		return err
+	}
+
+	log.Debugf("[Trello Migration] Got %d projects for board %s", len(board.Lists), board.ID)
+
+	listMap := make(map[string]*trello.List, len(board.Lists))
+	for _, list := range board.Lists {
+		listMap[list.ID] = list
+	}
+
+	log.Debugf("[Trello Migration] Getting cards for board %s", board.ID)
+
+	cards, err := board.GetCards(allArg)
+	if err != nil {
+		return
+	}
+
+	log.Debugf("[Trello Migration] Got %d cards for board %s", len(cards), board.ID)
+
+	for _, card := range cards {
+		list, exists := listMap[card.IDList]
+		if !exists {
+			continue
+		}
+
+		card.Attachments, err = card.GetAttachments(allArg)
 		if err != nil {
 			return
 		}
 
-		log.Debugf("[Trello Migration] Got %d projects for board %s", len(board.Lists), board.ID)
+		if len(card.IDCheckLists) > 0 {
-
+			for _, checkListID := range card.IDCheckLists {
-		listMap := make(map[string]*trello.List, len(board.Lists))
+				checklist, err := client.GetChecklist(checkListID, allArg)
-		for _, list := range board.Lists {
+				if err != nil {
-			listMap[list.ID] = list
+					return err
-		}
-
-		log.Debugf("[Trello Migration] Getting cards for board %s", board.ID)
-
-		cards, err := board.GetCards(allArg)
-		if err != nil {
-			return nil, err
-		}
-
-		log.Debugf("[Trello Migration] Got %d cards for board %s", len(cards), board.ID)
-
-		for _, card := range cards {
-			list, exists := listMap[card.IDList]
-			if !exists {
-				continue
-			}
-
-			card.Attachments, err = card.GetAttachments(allArg)
-			if err != nil {
-				return nil, err
-			}
-
-			if len(card.IDCheckLists) > 0 {
-				for _, checkListID := range card.IDCheckLists {
-					checklist, err := client.GetChecklist(checkListID, allArg)
-					if err != nil {
-						return nil, err
-					}
-
-					checklist.CheckItems = []trello.CheckItem{}
-					err = client.Get("checklists/"+checkListID+"/checkItems", allArg, &checklist.CheckItems)
-					if err != nil {
-						return nil, err
-					}
-
-					card.Checklists = append(card.Checklists, checklist)
-					log.Debugf("Retrieved checklist %s for card %s", checkListID, card.ID)
 				}
-			}
 
-			list.Cards = append(list.Cards, card)
+				checklist.CheckItems = []trello.CheckItem{}
+				err = client.Get("checklists/"+checkListID+"/checkItems", allArg, &checklist.CheckItems)
+				if err != nil {
+					return err
+				}
+
+				card.Checklists = append(card.Checklists, checklist)
+				log.Debugf("Retrieved checklist %s for card %s", checkListID, card.ID)
+			}
 		}
 
-		log.Debugf("[Trello Migration] Looked for attachements on all cards of board %s", board.ID)
+		list.Cards = append(list.Cards, card)
 	}
 
+	log.Debugf("[Trello Migration] Looked for attachements on all cards of board %s", board.ID)
+
 	return
 }
 
@@ -196,7 +216,7 @@ func convertMarkdownToHTML(input string) (output string, err error) {
 
 // Converts all previously obtained data from trello into the vikunja format.
 // `trelloData` should contain all boards with their projects and cards respectively.
-func convertTrelloDataToVikunja(trelloData []*trello.Board, token string) (fullVikunjaHierachie []*models.ProjectWithTasksAndBuckets, err error) {
+func convertTrelloDataToVikunja(organizationName string, trelloData []*trello.Board, token string) (fullVikunjaHierachie []*models.ProjectWithTasksAndBuckets, err error) {
 
 	log.Debugf("[Trello Migration] ")
 
@@ -205,7 +225,7 @@ func convertTrelloDataToVikunja(trelloData []*trello.Board, token string) (fullV
 		{
 			Project: models.Project{
 				ID:    pseudoParentID,
-				Title: "Imported from Trello",
+				Title: organizationName,
 			},
 		},
 	}
@@ -392,29 +412,58 @@ func (m *Migration) Migrate(u *user.User) (err error) {
 	log.Debugf("[Trello Migration] Starting migration for user %d", u.ID)
 	log.Debugf("[Trello Migration] Getting all trello data for user %d", u.ID)
 
-	trelloData, err := getTrelloData(m.Token)
+	client := trello.NewClient(config.MigrationTrelloKey.GetString(), m.Token)
+	client.Logger = log.GetLogger()
+
+	boards, err := getTrelloBoards(client)
 	if err != nil {
 		return
 	}
 
 	log.Debugf("[Trello Migration] Got all trello data for user %d", u.ID)
-	log.Debugf("[Trello Migration] Start converting trello data for user %d", u.ID)
 
-	fullVikunjaHierachie, err := convertTrelloDataToVikunja(trelloData, m.Token)
+	organizationMap := getTrelloOrganizationsWithBoards(boards)
-	if err != nil {
+	for organizationID, boards := range organizationMap {
-		return
+		log.Debugf("[Trello Migration] Getting organization with id %s for user %d", organizationID, u.ID)
+		orgName := organizationID
+		if organizationID != "Personal" {
+			organization, err := client.GetOrganization(organizationID, trello.Defaults())
+			if err != nil {
+				return err
+			}
+			orgName = organization.DisplayName
+		}
+
+		for _, board := range boards {
+			log.Debugf("[Trello Migration] Getting card data for board %s for user %d for organization %s", board.ID, u.ID, organizationID)
+
+			err = fillCardData(client, board)
+			if err != nil {
+				return err
+			}
+
+			log.Debugf("[Trello Migration] Got card data for board %s for user %d for organization %s", board.ID, u.ID, organizationID)
+		}
+
+		log.Debugf("[Trello Migration] Start converting trello data for user %d for organization %s", u.ID, organizationID)
+
+		hierarchy, err := convertTrelloDataToVikunja(orgName, boards, m.Token)
+		if err != nil {
+			return err
+		}
+
+		log.Debugf("[Trello Migration] Done migrating trello data for user %d for organization %s", u.ID, organizationID)
+		log.Debugf("[Trello Migration] Start inserting trello data for user %d for organization %s", u.ID, organizationID)
+
+		err = migration.InsertFromStructure(hierarchy, u)
+		if err != nil {
+			return err
+		}
+
+		log.Debugf("[Trello Migration] Done inserting trello data for user %d for organization %s", u.ID, organizationID)
 	}
 
-	log.Debugf("[Trello Migration] Done migrating trello data for user %d", u.ID)
+	log.Debugf("[Trello Migration] Done migrating all trello data for user %d", u.ID)
-	log.Debugf("[Trello Migration] Start inserting trello data for user %d", u.ID)
 
-	err = migration.InsertFromStructure(fullVikunjaHierachie, u)
+	return
-	if err != nil {
-		return
-	}
-
-	log.Debugf("[Trello Migration] Done inserting trello data for user %d", u.ID)
-	log.Debugf("[Trello Migration] Migration done for user %d", u.ID)
-
-	return nil
 }

pkg/modules/migration/trello/trello_test.go

@@ -32,20 +32,23 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestConvertTrelloToVikunja(t *testing.T) {
+func getTestBoard(t *testing.T) ([]*trello.Board, time.Time) {
 
 	config.InitConfig()
 
 	time1, err := time.Parse(time.RFC3339Nano, "2014-09-26T08:25:05Z")
 	require.NoError(t, err)
-	exampleFile, err := os.ReadFile(config.ServiceRootpath.GetString() + "/pkg/modules/migration/testimage.jpg")
-	require.NoError(t, err)
 
 	trelloData := []*trello.Board{
 		{
-			Name:   "TestBoard",
+			Name: "TestBoard",
-			Desc:   "This is a description",
+			Organization: trello.Organization{
-			Closed: false,
+				ID:          "orgid",
+				DisplayName: "TestOrg",
+			},
+			IDOrganization: "orgid",
+			Desc:           "This is a description",
+			Closed:         false,
 			Lists: []*trello.List{
 				{
 					Name: "Test Project 1",
@@ -168,8 +171,13 @@ func TestConvertTrelloToVikunja(t *testing.T) {
 			},
 		},
 		{
-			Name:   "TestBoard 2",
+			Organization: trello.Organization{
-			Closed: false,
+				ID:          "orgid2",
+				DisplayName: "TestOrg2",
+			},
+			IDOrganization: "orgid2",
+			Name:           "TestBoard 2",
+			Closed:         false,
 			Lists: []*trello.List{
 				{
 					Name: "Test Project 4",
@@ -183,8 +191,13 @@ func TestConvertTrelloToVikunja(t *testing.T) {
 			},
 		},
 		{
-			Name:   "TestBoard Archived",
+			Organization: trello.Organization{
-			Closed: true,
+				ID:          "orgid",
+				DisplayName: "TestOrg",
+			},
+			IDOrganization: "orgid",
+			Name:           "TestBoard Archived",
+			Closed:         true,
 			Lists: []*trello.List{
 				{
 					Name: "Test Project 5",
@@ -197,67 +210,91 @@ func TestConvertTrelloToVikunja(t *testing.T) {
 				},
 			},
 		},
+		{
+			Name: "Personal Board",
+			Lists: []*trello.List{
+				{
+					Name: "Test Project 6",
+					Cards: []*trello.Card{
+						{
+							Name: "Test Card 5659",
+							Pos:  123,
+						},
+					},
+				},
+			},
+		},
 	}
 	trelloData[0].Prefs.BackgroundImage = "https://vikunja.io/testimage.jpg" // Using an image which we are hosting, so it'll still be up
 
-	expectedHierachie := []*models.ProjectWithTasksAndBuckets{
+	return trelloData, time1
-		{
+}
-			Project: models.Project{
+
-				ID:    1,
+func TestConvertTrelloToVikunja(t *testing.T) {
-				Title: "Imported from Trello",
+	trelloData, time1 := getTestBoard(t)
-			},
+
-		},
+	exampleFile, err := os.ReadFile(config.ServiceRootpath.GetString() + "/pkg/modules/migration/testimage.jpg")
-		{
+	require.NoError(t, err)
-			Project: models.Project{
+
-				ID:                    2,
+	expectedHierarchyOrg := map[string][]*models.ProjectWithTasksAndBuckets{
-				ParentProjectID:       1,
+		"orgid": {
-				Title:                 "TestBoard",
+			{
-				Description:           "This is a description",
+				Project: models.Project{
-				BackgroundInformation: bytes.NewBuffer(exampleFile),
-			},
-			Buckets: []*models.Bucket{
-				{
 					ID:    1,
-					Title: "Test Project 1",
+					Title: "orgid",
-				},
-				{
-					ID:    2,
-					Title: "Test Project 2",
 				},
 			},
-			Tasks: []*models.TaskWithComments{
+			{
-				{
+				Project: models.Project{
-					Task: models.Task{
+					ID:                    2,
-						Title:       "Test Card 1",
+					ParentProjectID:       1,
-						Description: "<p>Card Description <strong>bold</strong></p>\n",
+					Title:                 "TestBoard",
-						BucketID:    1,
+					Description:           "This is a description",
-						DueDate:     time1,
+					BackgroundInformation: bytes.NewBuffer(exampleFile),
-						Labels: []*models.Label{
+				},
-							{
+				Buckets: []*models.Bucket{
-								Title:    "Label 1",
+					{
-								HexColor: trelloColorMap["green"],
+						ID:    1,
+						Title: "Test Project 1",
+					},
+					{
+						ID:    2,
+						Title: "Test Project 2",
+					},
+				},
+				Tasks: []*models.TaskWithComments{
+					{
+						Task: models.Task{
+							Title:       "Test Card 1",
+							Description: "<p>Card Description <strong>bold</strong></p>\n",
+							BucketID:    1,
+							DueDate:     time1,
+							Labels: []*models.Label{
+								{
+									Title:    "Label 1",
+									HexColor: trelloColorMap["green"],
+								},
+								{
+									Title:    "Label 2",
+									HexColor: trelloColorMap["orange"],
+								},
 							},
-							{
+							Attachments: []*models.TaskAttachment{
-								Title:    "Label 2",
+								{
-								HexColor: trelloColorMap["orange"],
+									File: &files.File{
-							},
+										Name:        "Testimage.jpg",
-						},
+										Mime:        "image/jpg",
-						Attachments: []*models.TaskAttachment{
+										Size:        uint64(len(exampleFile)),
-							{
+										FileContent: exampleFile,
-								File: &files.File{
+									},
-									Name:        "Testimage.jpg",
-									Mime:        "image/jpg",
-									Size:        uint64(len(exampleFile)),
-									FileContent: exampleFile,
 								},
 							},
 						},
 					},
-				},
+					{
-				{
+						Task: models.Task{
-					Task: models.Task{
+							Title: "Test Card 2",
-						Title: "Test Card 2",
+							Description: `
-						Description: `
 
 <h2> Checkproject 1</h2>
 
@@ -270,117 +307,180 @@ func TestConvertTrelloToVikunja(t *testing.T) {
 <ul data-type="taskList">
 <li data-checked="false" data-type="taskItem"><label><input type="checkbox"><span></span></label><div><p>Pending Task</p></div></li>
 <li data-checked="false" data-type="taskItem"><label><input type="checkbox"><span></span></label><div><p>Another Pending Task</p></div></li></ul>`,
-						BucketID: 1,
+							BucketID: 1,
+						},
 					},
-				},
+					{
-				{
+						Task: models.Task{
-					Task: models.Task{
+							Title:    "Test Card 3",
-						Title:    "Test Card 3",
+							BucketID: 1,
-						BucketID: 1,
+						},
 					},
-				},
+					{
-				{
+						Task: models.Task{
-					Task: models.Task{
+							Title:    "Test Card 4",
-						Title:    "Test Card 4",
+							BucketID: 1,
-						BucketID: 1,
+							Labels: []*models.Label{
-						Labels: []*models.Label{
+								{
-							{
+									Title:    "Label 2",
-								Title:    "Label 2",
+									HexColor: trelloColorMap["orange"],
-								HexColor: trelloColorMap["orange"],
+								},
 							},
 						},
 					},
-				},
+					{
-				{
+						Task: models.Task{
-					Task: models.Task{
+							Title:    "Test Card 5",
-						Title:    "Test Card 5",
+							BucketID: 2,
-						BucketID: 2,
+							Labels: []*models.Label{
-						Labels: []*models.Label{
+								{
-							{
+									Title:    "Label 3",
-								Title:    "Label 3",
+									HexColor: trelloColorMap["blue"],
-								HexColor: trelloColorMap["blue"],
+								},
-							},
+								{
-							{
+									Title:    "Label 4",
-								Title:    "Label 4",
+									HexColor: trelloColorMap["green_dark"],
-								HexColor: trelloColorMap["green_dark"],
+								},
-							},
+								{
-							{
+									Title:    "Label 5",
-								Title:    "Label 5",
+									HexColor: trelloColorMap["transparent"],
-								HexColor: trelloColorMap["transparent"],
+								},
 							},
 						},
 					},
-				},
+					{
-				{
+						Task: models.Task{
-					Task: models.Task{
+							Title:    "Test Card 6",
-						Title:    "Test Card 6",
+							BucketID: 2,
-						BucketID: 2,
+							DueDate:  time1,
-						DueDate:  time1,
+						},
+					},
+					{
+						Task: models.Task{
+							Title:    "Test Card 7",
+							BucketID: 2,
+						},
+					},
+					{
+						Task: models.Task{
+							Title:    "Test Card 8",
+							BucketID: 2,
+						},
 					},
 				},
-				{
+			},
-					Task: models.Task{
+			{
-						Title:    "Test Card 7",
+				Project: models.Project{
-						BucketID: 2,
+					ID:              3,
+					ParentProjectID: 1,
+					Title:           "TestBoard Archived",
+					IsArchived:      true,
+				},
+				Buckets: []*models.Bucket{
+					{
+						ID:    3,
+						Title: "Test Project 5",
 					},
 				},
-				{
+				Tasks: []*models.TaskWithComments{
-					Task: models.Task{
+					{
-						Title:    "Test Card 8",
+						Task: models.Task{
-						BucketID: 2,
+							Title:    "Test Card 63423",
+							BucketID: 3,
+						},
 					},
 				},
 			},
 		},
-		{
+		"orgid2": {
-			Project: models.Project{
+			{
-				ID:              3,
+				Project: models.Project{
-				ParentProjectID: 1,
+					ID:    1,
-				Title:           "TestBoard 2",
+					Title: "orgid2",
-			},
-			Buckets: []*models.Bucket{
-				{
-					ID:    3,
-					Title: "Test Project 4",
 				},
 			},
-			Tasks: []*models.TaskWithComments{
+			{
-				{
+				Project: models.Project{
-					Task: models.Task{
+					ID:              2,
-						Title:    "Test Card 634",
+					ParentProjectID: 1,
-						BucketID: 3,
+					Title:           "TestBoard 2",
+				},
+				Buckets: []*models.Bucket{
+					{
+						ID:    1,
+						Title: "Test Project 4",
+					},
+				},
+				Tasks: []*models.TaskWithComments{
+					{
+						Task: models.Task{
+							Title:    "Test Card 634",
+							BucketID: 1,
+						},
 					},
 				},
 			},
 		},
-		{
+		"Personal": {
-			Project: models.Project{
+			{
-				ID:              4,
+				Project: models.Project{
-				ParentProjectID: 1,
+					ID:    1,
-				Title:           "TestBoard Archived",
+					Title: "Personal",
-				IsArchived:      true,
-			},
-			Buckets: []*models.Bucket{
-				{
-					ID:    4,
-					Title: "Test Project 5",
 				},
 			},
-			Tasks: []*models.TaskWithComments{
+			{
-				{
+				Project: models.Project{
-					Task: models.Task{
+					ID:              2,
-						Title:    "Test Card 63423",
+					ParentProjectID: 1,
-						BucketID: 4,
+					Title:           "Personal Board",
+				},
+				Buckets: []*models.Bucket{
+					{
+						ID:    1,
+						Title: "Test Project 6",
+					},
+				},
+				Tasks: []*models.TaskWithComments{
+					{
+						Task: models.Task{
+							Title:    "Test Card 5659",
+							BucketID: 1,
+						},
 					},
 				},
 			},
 		},
 	}
 
-	hierachie, err := convertTrelloDataToVikunja(trelloData, "")
+	organizationMap := getTrelloOrganizationsWithBoards(trelloData)
-	require.NoError(t, err)
+	for organizationID, boards := range organizationMap {
-	assert.NotNil(t, hierachie)
+		hierarchy, err := convertTrelloDataToVikunja(organizationID, boards, "")
-	if diff, equal := messagediff.PrettyDiff(hierachie, expectedHierachie); !equal {
+
-		t.Errorf("converted trello data = %v, want %v, diff: %v", hierachie, expectedHierachie, diff)
+		require.NoError(t, err)
+		assert.NotNil(t, hierarchy)
+		if diff, equal := messagediff.PrettyDiff(hierarchy, expectedHierarchyOrg[organizationID]); !equal {
+			t.Errorf("converted trello data = %v,\nwant %v,\ndiff: %v", hierarchy, expectedHierarchyOrg[organizationID], diff)
+		}
+	}
+}
+
+func TestCreateOrganizationMap(t *testing.T) {
+	trelloData, _ := getTestBoard(t)
+
+	organizationMap := getTrelloOrganizationsWithBoards(trelloData)
+	expectedMap := map[string][]*trello.Board{
+		"orgid": {
+			trelloData[0],
+			trelloData[2],
+		},
+		"orgid2": {
+			trelloData[1],
+		},
+		"Personal": {
+			trelloData[3],
+		},
+	}
+	if diff, equal := messagediff.PrettyDiff(organizationMap, expectedMap); !equal {
+		t.Errorf("converted trello data = %v,\nwant %v,\ndiff: %v", organizationMap, expectedMap, diff)
 	}
 }