feat(migration): migration from other services now happens in the background

go.mod

@@ -71,6 +71,7 @@ require (
 	golang.org/x/sync v0.5.0
 	golang.org/x/sys v0.14.0
 	golang.org/x/term v0.13.0
+	golang.org/x/text v0.13.0
 	gopkg.in/d4l3k/messagediff.v1 v1.2.1
 	gopkg.in/yaml.v3 v3.0.1
 	src.techknowlogick.com/xgo v1.7.1-0.20231019133136-ecfba3dfed5d
@@ -173,7 +174,6 @@ require (
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.13.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect

pkg/initialize/init.go

@@ -29,6 +29,7 @@ import (
 	"code.vikunja.io/api/pkg/models"
 	"code.vikunja.io/api/pkg/modules/auth/openid"
 	"code.vikunja.io/api/pkg/modules/keyvalue"
+	migrationHandler "code.vikunja.io/api/pkg/modules/migration/handler"
 	"code.vikunja.io/api/pkg/red"
 	"code.vikunja.io/api/pkg/user"
 )
@@ -100,6 +101,7 @@ func FullInit() {
 	go func() {
 		models.RegisterListeners()
 		user.RegisterListeners()
+		migrationHandler.RegisterListeners()
 		err := events.InitEvents()
 		if err != nil {
 			log.Fatal(err.Error())

pkg/migration/20231108231513.go

@@ -0,0 +1,97 @@
+// Vikunja is a to-do list application to facilitate your life.
+// Copyright 2018-present Vikunja and contributors. All rights reserved.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public Licensee as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public Licensee for more details.
+//
+// You should have received a copy of the GNU Affero General Public Licensee
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package migration
+
+import (
+	"time"
+
+	"code.vikunja.io/api/pkg/config"
+	"src.techknowlogick.com/xormigrate"
+	"xorm.io/xorm"
+)
+
+type migrationStatus20231108231513 struct {
+	ID         int64     `xorm:"bigint autoincr not null unique pk" json:"id"`
+	Created    time.Time `xorm:"created not null 'created'" json:"time"`
+	StartedAt  time.Time `xorm:"null" json:"started_at"`
+	FinishedAt time.Time `xorm:"null" json:"finished_at"`
+}
+
+func (migrationStatus20231108231513) TableName() string {
+	return "migration_status"
+}
+
+func init() {
+	migrations = append(migrations, &xormigrate.Migration{
+		ID:          "20231108231513",
+		Description: "",
+		Migrate: func(tx *xorm.Engine) error {
+
+			err := tx.Sync2(migrationStatus20231108231513{})
+			if err != nil {
+				return err
+			}
+
+			all := []*migrationStatus20231108231513{}
+			err = tx.Find(&all)
+			if err != nil {
+				return err
+			}
+
+			for _, status := range all {
+				status.StartedAt = status.Created
+				status.FinishedAt = status.Created
+				_, err = tx.Where("id = ?", status.ID).Update(status)
+				if err != nil {
+					return err
+				}
+			}
+
+			if config.DatabaseType.GetString() == "sqlite" {
+				_, err = tx.Exec(`create table migration_status_dg_tmp
+				(
+					id            INTEGER not null
+				primary key autoincrement,
+					user_id       INTEGER not null,
+					migrator_name TEXT,
+					started_at DATETIME null,
+					finished_at DATETIME null
+				);
+
+				insert into migration_status_dg_tmp(id, user_id, migrator_name, started_at, finished_at)
+				select id, user_id, migrator_name, started_at, finished_at
+					from migration_status;
+
+					drop table migration_status;
+
+					alter table migration_status_dg_tmp
+					rename to migration_status;
+
+					create unique index UQE_migration_status_id
+					on migration_status (id);
+`)
+				return err
+			}
+
+			err = dropTableColum(tx, "migration_status", "created")
+			return err
+		},
+		Rollback: func(tx *xorm.Engine) error {
+			return nil
+		},
+	})
+}

pkg/modules/migration/handler/events.go

@@ -0,0 +1,33 @@
+// Vikunja is a to-do list application to facilitate your life.
+// Copyright 2018-present Vikunja and contributors. All rights reserved.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public Licensee as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public Licensee for more details.
+//
+// You should have received a copy of the GNU Affero General Public Licensee
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package handler
+
+import (
+	"code.vikunja.io/api/pkg/user"
+)
+
+// MigrationRequestedEvent represents a MigrationRequestedEvent event
+type MigrationRequestedEvent struct {
+	Migrator     interface{} `json:"migrator"`
+	User         *user.User  `json:"user"`
+	MigratorKind string      `json:"migrator_kind"`
+}
+
+// Name defines the name for MigrationRequestedEvent
+func (t *MigrationRequestedEvent) Name() string {
+	return "migration.requested"
+}

pkg/modules/migration/handler/handler.go

@@ -19,6 +19,7 @@ package handler
 import (
 	"net/http"
 
+	"code.vikunja.io/api/pkg/events"
 	"code.vikunja.io/api/pkg/models"
 	"code.vikunja.io/api/pkg/modules/migration"
 	user2 "code.vikunja.io/api/pkg/user"
@@ -26,6 +27,12 @@ import (
 	"github.com/labstack/echo/v4"
 )
 
+var registeredMigrators map[string]*MigrationWeb
+
+func init() {
+	registeredMigrators = make(map[string]*MigrationWeb)
+}
+
 // MigrationWeb holds the web migration handler
 type MigrationWeb struct {
 	MigrationStruct func() migration.Migrator
@@ -36,12 +43,13 @@ type AuthURL struct {
 	URL string `json:"url"`
 }
 
-// RegisterRoutes registers all routes for migration
+// RegisterMigrator registers all routes for migration
-func (mw *MigrationWeb) RegisterRoutes(g *echo.Group) {
+func (mw *MigrationWeb) RegisterMigrator(g *echo.Group) {
 	ms := mw.MigrationStruct()
 	g.GET("/"+ms.Name()+"/auth", mw.AuthURL)
 	g.GET("/"+ms.Name()+"/status", mw.Status)
 	g.POST("/"+ms.Name()+"/migrate", mw.Migrate)
+	registeredMigrators[ms.Name()] = mw
 }
 
 // AuthURL is the web handler to get the auth url
@@ -60,19 +68,29 @@ func (mw *MigrationWeb) Migrate(c echo.Context) error {
 		return handler.HandleHTTPError(err, c)
 	}
 
+	stats, err := migration.GetMigrationStatus(ms, user)
+	if err != nil {
+		return handler.HandleHTTPError(err, c)
+	}
+
+	if stats.FinishedAt.IsZero() {
+		return c.JSON(http.StatusOK, map[string]string{
+			"message":       "Migration already running",
+			"running_since": stats.StartedAt.String(),
+		})
+	}
+
 	// Bind user request stuff
 	err = c.Bind(ms)
 	if err != nil {
 		return echo.NewHTTPError(http.StatusBadRequest, "No or invalid model provided: "+err.Error())
 	}
 
-	// Do the migration
+	err = events.Dispatch(&MigrationRequestedEvent{
-	err = ms.Migrate(user)
+		Migrator:     ms,
-	if err != nil {
+		MigratorKind: ms.Name(),
-		return handler.HandleHTTPError(err, c)
+		User:         user,
-	}
+	})
-
-	err = migration.SetMigrationStatus(ms, user)
 	if err != nil {
 		return handler.HandleHTTPError(err, c)
 	}

pkg/modules/migration/handler/handler_file.go

@@ -57,13 +57,18 @@ func (fw *FileMigratorWeb) Migrate(c echo.Context) error {
 	}
 	defer src.Close()
 
+	m, err := migration.StartMigration(ms, user)
+	if err != nil {
+		return handler.HandleHTTPError(err, c)
+	}
+
 	// Do the migration
 	err = ms.Migrate(user, src, file.Size)
 	if err != nil {
 		return handler.HandleHTTPError(err, c)
 	}
 
-	err = migration.SetMigrationStatus(ms, user)
+	err = migration.FinishMigration(m)
 	if err != nil {
 		return handler.HandleHTTPError(err, c)
 	}

pkg/modules/migration/handler/listeners.go

@@ -0,0 +1,88 @@
+// Vikunja is a to-do list application to facilitate your life.
+// Copyright 2018-present Vikunja and contributors. All rights reserved.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public Licensee as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public Licensee for more details.
+//
+// You should have received a copy of the GNU Affero General Public Licensee
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package handler
+
+import (
+	"encoding/json"
+
+	"code.vikunja.io/api/pkg/events"
+	"code.vikunja.io/api/pkg/log"
+	"code.vikunja.io/api/pkg/modules/migration"
+	"code.vikunja.io/api/pkg/notifications"
+
+	"github.com/ThreeDotsLabs/watermill/message"
+)
+
+func RegisterListeners() {
+	events.RegisterListener((&MigrationRequestedEvent{}).Name(), &MigrationListener{})
+}
+
+// MigrationListener  represents a listener
+type MigrationListener struct {
+}
+
+// Name defines the name for the MigrationListener listener
+func (s *MigrationListener) Name() string {
+	return "migration.listener"
+}
+
+// Handle is executed when the event MigrationListener listens on is fired
+func (s *MigrationListener) Handle(msg *message.Message) (err error) {
+	event := &MigrationRequestedEvent{}
+	err = json.Unmarshal(msg.Payload, event)
+	if err != nil {
+		return
+	}
+
+	mstr := registeredMigrators[event.MigratorKind]
+	event.Migrator = mstr.MigrationStruct()
+
+	// unmarshaling again to make sure the migrator has the correct type now
+	err = json.Unmarshal(msg.Payload, event)
+	if err != nil {
+		return
+	}
+
+	ms := event.Migrator.(migration.Migrator)
+
+	m, err := migration.StartMigration(ms, event.User)
+	if err != nil {
+		return
+	}
+
+	log.Debugf("[Migration] Starting migration %d from %s for user %d", m.ID, event.MigratorKind, event.User.ID)
+
+	err = ms.Migrate(event.User)
+	if err != nil {
+		return
+	}
+
+	err = migration.FinishMigration(m)
+	if err != nil {
+		return
+	}
+
+	err = notifications.Notify(event.User, &MigrationDoneNotification{
+		MigratorName: ms.Name(),
+	})
+	if err != nil {
+		return
+	}
+
+	log.Debugf("[Migration] Successfully done migration %d from %s for user %d", m.ID, event.MigratorKind, event.User.ID)
+	return
+}

pkg/modules/migration/handler/notifications.go

@@ -0,0 +1,51 @@
+// Vikunja is a to-do list application to facilitate your life.
+// Copyright 2018-present Vikunja and contributors. All rights reserved.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public Licensee as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public Licensee for more details.
+//
+// You should have received a copy of the GNU Affero General Public Licensee
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package handler
+
+import (
+	"golang.org/x/text/cases"
+	"golang.org/x/text/language"
+
+	"code.vikunja.io/api/pkg/config"
+	"code.vikunja.io/api/pkg/notifications"
+)
+
+// MigrationDoneNotification represents a MigrationDoneNotification notification
+type MigrationDoneNotification struct {
+	MigratorName string
+}
+
+// ToMail returns the mail notification for MigrationDoneNotification
+func (n *MigrationDoneNotification) ToMail() *notifications.Mail {
+	kind := cases.Title(language.English).String(n.MigratorName)
+
+	return notifications.NewMail().
+		Subject("The migration from "+kind+" to Vikunja was completed").
+		Line("Vikunja has imported all lists/projects, tasks, notes, reminders and files from "+kind+" you have access to.").
+		Action("View your imported projects in Vikunja", config.ServiceFrontendurl.GetString()).
+		Line("Have fun with your new (old) projects!")
+}
+
+// ToDB returns the MigrationDoneNotification notification in a format which can be saved in the db
+func (n *MigrationDoneNotification) ToDB() interface{} {
+	return nil
+}
+
+// Name returns the name of the notification
+func (n *MigrationDoneNotification) Name() string {
+	return "migration.done"
+}

pkg/modules/migration/migration_status.go

@@ -28,7 +28,8 @@ type Status struct {
 	ID           int64     `xorm:"bigint autoincr not null unique pk" json:"id"`
 	UserID       int64     `xorm:"bigint not null" json:"-"`
 	MigratorName string    `xorm:"varchar(255)" json:"migrator_name"`
-	Created      time.Time `xorm:"created not null 'created'" json:"time"`
+	StartedAt    time.Time `xorm:"not null" json:"started_at"`
+	FinishedAt   time.Time `xorm:"null" json:"finished_at"`
 }
 
 // TableName holds the table name for the migration status table
@@ -36,19 +37,31 @@ func (s *Status) TableName() string {
 	return "migration_status"
 }
 
-// SetMigrationStatus sets the migration status for a user
+// StartMigration sets the migration status for a user
-func SetMigrationStatus(m MigratorName, u *user.User) (err error) {
+func StartMigration(m MigratorName, u *user.User) (status *Status, err error) {
 	s := db.NewSession()
 	defer s.Close()
 
-	status := &Status{
+	status = &Status{
 		UserID:       u.ID,
 		MigratorName: m.Name(),
+		StartedAt:    time.Now(),
 	}
 	_, err = s.Insert(status)
 	return
 }
 
+// FinishMigration sets the finished at time and calls it a day
+func FinishMigration(status *Status) (err error) {
+	s := db.NewSession()
+	defer s.Close()
+
+	status.FinishedAt = time.Now()
+
+	_, err = s.Where("id = ?", status.ID).Update(status)
+	return
+}
+
 // GetMigrationStatus returns the migration status for a migration and a user
 func GetMigrationStatus(m MigratorName, u *user.User) (status *Status, err error) {
 	s := db.NewSession()

pkg/routes/routes.go

@@ -599,7 +599,7 @@ func registerMigrations(m *echo.Group) {
 				return &todoist.Migration{}
 			},
 		}
-		todoistMigrationHandler.RegisterRoutes(m)
+		todoistMigrationHandler.RegisterMigrator(m)
 	}
 
 	// Trello
@@ -609,7 +609,7 @@ func registerMigrations(m *echo.Group) {
 				return &trello.Migration{}
 			},
 		}
-		trelloMigrationHandler.RegisterRoutes(m)
+		trelloMigrationHandler.RegisterMigrator(m)
 	}
 
 	// Microsoft Todo
@@ -619,7 +619,7 @@ func registerMigrations(m *echo.Group) {
 				return &microsofttodo.Migration{}
 			},
 		}
-		microsoftTodoMigrationHandler.RegisterRoutes(m)
+		microsoftTodoMigrationHandler.RegisterMigrator(m)
 	}
 
 	// Vikunja File Migrator