User Data Export and import (#967)

Co-authored-by: kolaente <k@knt.li>
Reviewed-on: https://kolaente.dev/vikunja/api/pulls/967
Co-authored-by: konrad <k@knt.li>
Co-committed-by: konrad <k@knt.li>

pkg/models/events.go

@@ -21,6 +21,16 @@ import (
 	"code.vikunja.io/web"
 )
 
+// DataExportRequestEvent represents a DataExportRequestEvent event
+type DataExportRequestEvent struct {
+	User *user.User
+}
+
+// Name defines the name for DataExportRequestEvent
+func (t *DataExportRequestEvent) Name() string {
+	return "user.export.request"
+}
+
 /////////////////
 // Task Events //
 /////////////////
@@ -257,3 +267,13 @@ type TeamDeletedEvent struct {
 func (t *TeamDeletedEvent) Name() string {
 	return "team.deleted"
 }
+
+// UserDataExportRequestedEvent represents a UserDataExportRequestedEvent event
+type UserDataExportRequestedEvent struct {
+	User *user.User
+}
+
+// Name defines the name for UserDataExportRequestedEvent
+func (t *UserDataExportRequestedEvent) Name() string {
+	return "user.export.requested"
+}

pkg/models/export.go

@@ -0,0 +1,358 @@
+// Vikunja is a to-do list application to facilitate your life.
+// Copyright 2018-2021 Vikunja and contributors. All rights reserved.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public Licensee as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public Licensee for more details.
+//
+// You should have received a copy of the GNU Affero General Public Licensee
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package models
+
+import (
+	"archive/zip"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"time"
+
+	"code.vikunja.io/api/pkg/config"
+	"code.vikunja.io/api/pkg/cron"
+	"code.vikunja.io/api/pkg/db"
+	"code.vikunja.io/api/pkg/files"
+	"code.vikunja.io/api/pkg/log"
+	"code.vikunja.io/api/pkg/notifications"
+	"code.vikunja.io/api/pkg/user"
+	"code.vikunja.io/api/pkg/utils"
+	"code.vikunja.io/api/pkg/version"
+	"xorm.io/xorm"
+)
+
+func ExportUserData(s *xorm.Session, u *user.User) (err error) {
+	exportDir := config.ServiceRootpath.GetString() + "/files/user-export-tmp/"
+	err = os.MkdirAll(exportDir, 0700)
+	if err != nil {
+		return err
+	}
+
+	tmpFilename := exportDir + strconv.FormatInt(u.ID, 10) + "_" + time.Now().Format("2006-01-02_15-03-05") + ".zip"
+
+	// Open zip
+	dumpFile, err := os.Create(tmpFilename)
+	if err != nil {
+		return fmt.Errorf("error opening dump file: %s", err)
+	}
+	defer dumpFile.Close()
+
+	dumpWriter := zip.NewWriter(dumpFile)
+	defer dumpWriter.Close()
+
+	// Get the data
+	err = exportListsAndTasks(s, u, dumpWriter)
+	if err != nil {
+		return err
+	}
+	// Task attachment files
+	err = exportTaskAttachments(s, u, dumpWriter)
+	if err != nil {
+		return err
+	}
+	// Saved filters
+	err = exportSavedFilters(s, u, dumpWriter)
+	if err != nil {
+		return err
+	}
+	// Background files
+	err = exportListBackgrounds(s, u, dumpWriter)
+	if err != nil {
+		return err
+	}
+	// Vikunja Version
+	err = utils.WriteBytesToZip("VERSION", []byte(version.Version), dumpWriter)
+	if err != nil {
+		return err
+	}
+
+	// If we reuse the same file again, saving it as a file in Vikunja will save it as a file with 0 bytes in size.
+	// Closing and reopening does work.
+	dumpWriter.Close()
+	dumpFile.Close()
+
+	exported, err := os.Open(tmpFilename)
+	if err != nil {
+		return err
+	}
+
+	stat, err := exported.Stat()
+	if err != nil {
+		return err
+	}
+
+	exportFile, err := files.CreateWithMimeAndSession(s, exported, tmpFilename, uint64(stat.Size()), u, "application/zip")
+	if err != nil {
+		return err
+	}
+
+	// Save the file id with the user
+	u.ExportFileID = exportFile.ID
+	_, err = s.Cols("export_file_id").Update(u)
+	if err != nil {
+		return
+	}
+
+	// Remove the old file
+	err = os.Remove(exported.Name())
+	if err != nil {
+		return err
+	}
+
+	// Send a notification
+	return notifications.Notify(u, &DataExportReadyNotification{
+		User: u,
+	})
+}
+
+func exportListsAndTasks(s *xorm.Session, u *user.User, wr *zip.Writer) (err error) {
+
+	namspaces, _, _, err := (&Namespace{}).ReadAll(s, u, "", -1, 0)
+	if err != nil {
+		return err
+	}
+
+	namespaceIDs := []int64{}
+	namespaces := []*NamespaceWithListsAndTasks{}
+	for _, n := range namspaces.([]*NamespaceWithLists) {
+		if n.ID < 1 {
+			// Don't include filters
+			continue
+		}
+
+		nn := &NamespaceWithListsAndTasks{
+			Namespace: n.Namespace,
+			Lists:     []*ListWithTasksAndBuckets{},
+		}
+
+		for _, l := range n.Lists {
+			nn.Lists = append(nn.Lists, &ListWithTasksAndBuckets{
+				List:             *l,
+				BackgroundFileID: l.BackgroundFileID,
+				Tasks:            []*TaskWithComments{},
+			})
+		}
+
+		namespaceIDs = append(namespaceIDs, n.ID)
+		namespaces = append(namespaces, nn)
+	}
+
+	if len(namespaceIDs) == 0 {
+		return nil
+	}
+
+	// Get all lists
+	lists, err := getListsForNamespaces(s, namespaceIDs, true)
+	if err != nil {
+		return err
+	}
+
+	tasks, _, _, err := getTasksForLists(s, lists, u, &taskOptions{
+		page:    0,
+		perPage: -1,
+	})
+	if err != nil {
+		return err
+	}
+
+	listMap := make(map[int64]*ListWithTasksAndBuckets)
+	listIDs := []int64{}
+	for _, n := range namespaces {
+		for _, l := range n.Lists {
+			listMap[l.ID] = l
+			listIDs = append(listIDs, l.ID)
+		}
+	}
+
+	taskMap := make(map[int64]*TaskWithComments, len(tasks))
+	for _, t := range tasks {
+		taskMap[t.ID] = &TaskWithComments{
+			Task: *t,
+		}
+		listMap[t.ListID].Tasks = append(listMap[t.ListID].Tasks, taskMap[t.ID])
+	}
+
+	comments := []*TaskComment{}
+	err = s.
+		Join("LEFT", "tasks", "tasks.id = task_comments.task_id").
+		In("tasks.list_id", listIDs).
+		Find(&comments)
+	if err != nil {
+		return
+	}
+
+	for _, c := range comments {
+		taskMap[c.TaskID].Comments = append(taskMap[c.TaskID].Comments, c)
+	}
+
+	buckets := []*Bucket{}
+	err = s.In("list_id", listIDs).Find(&buckets)
+	if err != nil {
+		return
+	}
+
+	for _, b := range buckets {
+		listMap[b.ListID].Buckets = append(listMap[b.ListID].Buckets, b)
+	}
+
+	data, err := json.Marshal(namespaces)
+	if err != nil {
+		return err
+	}
+
+	return utils.WriteBytesToZip("data.json", data, wr)
+}
+
+func exportTaskAttachments(s *xorm.Session, u *user.User, wr *zip.Writer) (err error) {
+	lists, _, _, err := getRawListsForUser(
+		s,
+		&listOptions{
+			user: u,
+			page: -1,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	tasks, _, _, err := getRawTasksForLists(s, lists, u, &taskOptions{page: -1})
+	if err != nil {
+		return err
+	}
+
+	taskIDs := []int64{}
+	for _, t := range tasks {
+		taskIDs = append(taskIDs, t.ID)
+	}
+
+	tas, err := getTaskAttachmentsByTaskIDs(s, taskIDs)
+	if err != nil {
+		return err
+	}
+
+	fs := make(map[int64]io.ReadCloser)
+	for _, ta := range tas {
+		if err := ta.File.LoadFileByID(); err != nil {
+			return err
+		}
+		fs[ta.FileID] = ta.File.File
+	}
+
+	return utils.WriteFilesToZip(fs, wr)
+}
+
+func exportSavedFilters(s *xorm.Session, u *user.User, wr *zip.Writer) (err error) {
+	filters, err := getSavedFiltersForUser(s, u)
+	if err != nil {
+		return err
+	}
+
+	data, err := json.Marshal(filters)
+	if err != nil {
+		return err
+	}
+
+	return utils.WriteBytesToZip("filters.json", data, wr)
+}
+
+func exportListBackgrounds(s *xorm.Session, u *user.User, wr *zip.Writer) (err error) {
+	lists, _, _, err := getRawListsForUser(
+		s,
+		&listOptions{
+			user: u,
+			page: -1,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	fs := make(map[int64]io.ReadCloser)
+	for _, l := range lists {
+		if l.BackgroundFileID == 0 {
+			continue
+		}
+
+		bgFile := &files.File{
+			ID: l.BackgroundFileID,
+		}
+		err = bgFile.LoadFileByID()
+		if err != nil {
+			return
+		}
+
+		fs[l.BackgroundFileID] = bgFile.File
+	}
+
+	return utils.WriteFilesToZip(fs, wr)
+}
+
+func RegisterOldExportCleanupCron() {
+	const logPrefix = "[User Export Cleanup Cron] "
+
+	err := cron.Schedule("0 * * * *", func() {
+		s := db.NewSession()
+		defer s.Close()
+
+		users := []*user.User{}
+		err := s.Where("export_file_id IS NOT NULL AND export_file_id != ?", 0).Find(&users)
+		if err != nil {
+			log.Errorf(logPrefix+"Could not get users with export files: %s", err)
+			return
+		}
+
+		fileIDs := []int64{}
+		for _, u := range users {
+			fileIDs = append(fileIDs, u.ExportFileID)
+		}
+
+		fs := []*files.File{}
+		err = s.Where("created < ?", time.Now().Add(-time.Hour*24*7)).In("id", fileIDs).Find(&fs)
+		if err != nil {
+			log.Errorf(logPrefix+"Could not get users with export files: %s", err)
+			return
+		}
+
+		if len(fs) == 0 {
+			return
+		}
+
+		log.Debugf(logPrefix+"Removing %d old user data exports...", len(fs))
+
+		for _, f := range fs {
+			err = f.Delete()
+			if err != nil {
+				log.Errorf(logPrefix+"Could not remove user export file %d: %s", f.ID, err)
+				return
+			}
+		}
+
+		_, err = s.In("export_file_id", fileIDs).Cols("export_file_id").Update(&user.User{})
+		if err != nil {
+			log.Errorf(logPrefix+"Could not update user export file state: %s", err)
+			return
+		}
+
+		log.Debugf(logPrefix+"Removed %d old user data exports...", len(fs))
+
+	})
+	if err != nil {
+		log.Fatalf("Could not old export cleanup cron: %s", err)
+	}
+}

pkg/models/list.go

@@ -51,12 +51,6 @@ type List struct {
 
 	// The user who created this list.
 	Owner *user.User `xorm:"-" json:"owner" valid:"-"`
-	// An array of tasks which belong to the list.
-	// Deprecated: you should use the dedicated task list endpoint because it has support for pagination and filtering
-	Tasks []*Task `xorm:"-" json:"-"`
-
-	// Only used for migration.
-	Buckets []*Bucket `xorm:"-" json:"-"`
 
 	// Whether or not a list is archived.
 	IsArchived bool `xorm:"not null default false" json:"is_archived" query:"is_archived"`
@@ -85,6 +79,15 @@ type List struct {
 	web.Rights   `xorm:"-" json:"-"`
 }
 
+type ListWithTasksAndBuckets struct {
+	List
+	// An array of tasks which belong to the list.
+	Tasks []*TaskWithComments `xorm:"-" json:"tasks"`
+	// Only used for migration.
+	Buckets          []*Bucket `xorm:"-" json:"buckets"`
+	BackgroundFileID int64     `xorm:"null" json:"background_file_id"`
+}
+
 // TableName returns a better name for the lists table
 func (l *List) TableName() string {
 	return "lists"

pkg/models/listeners.go

@@ -50,6 +50,7 @@ func RegisterListeners() {
 	events.RegisterListener((&TaskCommentUpdatedEvent{}).Name(), &HandleTaskCommentEditMentions{})
 	events.RegisterListener((&TaskCreatedEvent{}).Name(), &HandleTaskCreateMentions{})
 	events.RegisterListener((&TaskUpdatedEvent{}).Name(), &HandleTaskUpdatedMentions{})
+	events.RegisterListener((&UserDataExportRequestedEvent{}).Name(), &HandleUserDataExport{})
 }
 
 //////
@@ -562,3 +563,41 @@ func (s *SendTeamMemberAddedNotification) Handle(msg *message.Message) (err erro
 		Team:   event.Team,
 	})
 }
+
+// HandleUserDataExport  represents a listener
+type HandleUserDataExport struct {
+}
+
+// Name defines the name for the HandleUserDataExport listener
+func (s *HandleUserDataExport) Name() string {
+	return "handle.user.data.export"
+}
+
+// Handle is executed when the event HandleUserDataExport listens on is fired
+func (s *HandleUserDataExport) Handle(msg *message.Message) (err error) {
+	event := &UserDataExportRequestedEvent{}
+	err = json.Unmarshal(msg.Payload, event)
+	if err != nil {
+		return err
+	}
+
+	log.Debugf("Starting to export user data for user %d...", event.User.ID)
+
+	sess := db.NewSession()
+	defer sess.Close()
+	err = sess.Begin()
+	if err != nil {
+		return
+	}
+
+	err = ExportUserData(sess, event.User)
+	if err != nil {
+		_ = sess.Rollback()
+		return
+	}
+
+	log.Debugf("Done exporting user data for user %d...", event.User.ID)
+
+	err = sess.Commit()
+	return err
+}

pkg/models/namespace.go

@@ -187,6 +187,11 @@ type NamespaceWithLists struct {
 	Lists     []*List `xorm:"-" json:"lists"`
 }
 
+type NamespaceWithListsAndTasks struct {
+	Namespace
+	Lists []*ListWithTasksAndBuckets `xorm:"-" json:"lists"`
+}
+
 func makeNamespaceSlice(namespaces map[int64]*NamespaceWithLists, userMap map[int64]*user.User, subscriptions map[int64]*Subscription) []*NamespaceWithLists {
 	all := make([]*NamespaceWithLists, 0, len(namespaces))
 	for _, n := range namespaces {

pkg/models/notifications.go

@@ -302,3 +302,29 @@ func (n *UserMentionedInTaskNotification) ToDB() interface{} {
 func (n *UserMentionedInTaskNotification) Name() string {
 	return "task.mentioned"
 }
+
+// DataExportReadyNotification represents a DataExportReadyNotification notification
+type DataExportReadyNotification struct {
+	User *user.User `json:"user"`
+}
+
+// ToMail returns the mail notification for DataExportReadyNotification
+func (n *DataExportReadyNotification) ToMail() *notifications.Mail {
+	return notifications.NewMail().
+		Subject("Your Vikunja Data Export is ready").
+		Greeting("Hi "+n.User.GetName()+",").
+		Line("Your Vikunja Data Export is ready for you to download. Click the button below to download it:").
+		Action("Download", config.ServiceFrontendurl.GetString()+"user/export/download").
+		Line("The download will be available for the next 7 days.").
+		Line("Have a nice day!")
+}
+
+// ToDB returns the DataExportReadyNotification notification in a format which can be saved in the db
+func (n *DataExportReadyNotification) ToDB() interface{} {
+	return nil
+}
+
+// Name returns the name of the notification
+func (n *DataExportReadyNotification) Name() string {
+	return "data.export.ready"
+}

pkg/models/tasks.go

@@ -129,6 +129,11 @@ type Task struct {
 	web.Rights   `xorm:"-" json:"-"`
 }
 
+type TaskWithComments struct {
+	Task
+	Comments []*TaskComment `xorm:"-" json:"comments"`
+}
+
 // TableName returns the table name for listtasks
 func (Task) TableName() string {
 	return "tasks"