ami/tl-vikunja
1
0
Fork 0
Code Activity

User Data Export and import (#967)

Browse Source 
Co-authored-by: kolaente <k@knt.li>
Reviewed-on: https://kolaente.dev/vikunja/api/pulls/967
Co-authored-by: konrad <k@knt.li>
Co-committed-by: konrad <k@knt.li>
This commit is contained in:
konrad
2021-09-04 19:26:31 +00:00
parent fc51a3e76f
commit 90146aea5b
46 changed files with 2395 additions and 582 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
pkg/modules/migration/vikunja-file/export.zip Normal file
View File

Binary file not shown.

44
pkg/modules/migration/vikunja-file/main_test.go Normal file
View File

@ -0,0 +1,44 @@
// Vikunja is a to-do list application to facilitate your life.
// Copyright 2018-2021 Vikunja and contributors. All rights reserved.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public Licensee as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public Licensee for more details.
//
// You should have received a copy of the GNU Affero General Public Licensee
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package vikunjafile
import (
"os"
"testing"
"code.vikunja.io/api/pkg/events"
"code.vikunja.io/api/pkg/config"
"code.vikunja.io/api/pkg/files"
"code.vikunja.io/api/pkg/models"
"code.vikunja.io/api/pkg/user"
)
// TestMain is the main test function used to bootstrap the test env
func TestMain(m *testing.M) {
// Set default config
config.InitDefaultConfig()
// We need to set the root path even if we're not using the config, otherwise fixtures are not loaded correctly
config.ServiceRootpath.Set(os.Getenv("VIKUNJA_SERVICE_ROOTPATH"))
// Some tests use the file engine, so we'll need to initialize that
files.InitTests()
user.InitTests()
models.SetupTests()
events.Fake()
os.Exit(m.Run())
}

204
pkg/modules/migration/vikunja-file/vikunja.go Normal file
View File

@ -0,0 +1,204 @@
// Vikunja is a to-do list application to facilitate your life.
// Copyright 2018-2021 Vikunja and contributors. All rights reserved.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public Licensee as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public Licensee for more details.
//
// You should have received a copy of the GNU Affero General Public Licensee
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package vikunjafile
import (
"archive/zip"
"bytes"
"encoding/json"
"fmt"
"io"
"strconv"
"strings"
"code.vikunja.io/api/pkg/db"
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/models"
"code.vikunja.io/api/pkg/modules/migration"
"code.vikunja.io/api/pkg/user"
)
const logPrefix = "[Vikunja File Import] "
type FileMigrator struct {
}
// Name is used to get the name of the vikunja-file migration - we're using the docs here to annotate the status route.
// @Summary Get migration status
// @Description Returns if the current user already did the migation or not. This is useful to show a confirmation message in the frontend if the user is trying to do the same migration again.
// @tags migration
// @Produce json
// @Security JWTKeyAuth
// @Success 200 {object} migration.Status "The migration status"
// @Failure 500 {object} models.Message "Internal server error"
// @Router /migration/vikunja-file/status [get]
func (v *FileMigrator) Name() string {
return "vikunja-file"
}
// Migrate takes a vikunja file export, parses it and imports everything in it into Vikunja.
// @Summary Import all lists, tasks etc. from a Vikunja data export
// @Description Imports all projects, tasks, notes, reminders, subtasks and files from a Vikunjda data export into Vikunja.
// @tags migration
// @Accept json
// @Produce json
// @Security JWTKeyAuth
// @Param import formData string true "The Vikunja export zip file."
// @Success 200 {object} models.Message "A message telling you everything was migrated successfully."
// @Failure 500 {object} models.Message "Internal server error"
// @Router /migration/vikunja-file/migrate [post]
func (v *FileMigrator) Migrate(user *user.User, file io.ReaderAt, size int64) error {
r, err := zip.NewReader(file, size)
if err != nil {
return fmt.Errorf("could not open import file: %s", err)
}
log.Debugf(logPrefix+"Importing a zip file containing %d files", len(r.File))
var dataFile *zip.File
var filterFile *zip.File
storedFiles := make(map[int64]*zip.File)
for _, f := range r.File {
if strings.HasPrefix(f.Name, "files/") {
fname := strings.ReplaceAll(f.Name, "files/", "")
id, err := strconv.ParseInt(fname, 10, 64)
if err != nil {
return fmt.Errorf("could not convert file id: %s", err)
}
storedFiles[id] = f
log.Debugf(logPrefix + "Found a blob file")
continue
}
if f.Name == "data.json" {
dataFile = f
log.Debugf(logPrefix + "Found a data file")
continue
}
if f.Name == "filters.json" {
filterFile = f
log.Debugf(logPrefix + "Found a filter file")
}
}
if dataFile == nil {
return fmt.Errorf("no data file provided")
}
log.Debugf(logPrefix + "")
//////
// Import the bulk of Vikunja data
df, err := dataFile.Open()
if err != nil {
return fmt.Errorf("could not open data file: %s", err)
}
defer df.Close()
var bufData bytes.Buffer
if _, err := bufData.ReadFrom(df); err != nil {
return fmt.Errorf("could not read data file: %s", err)
}
namespaces := []*models.NamespaceWithListsAndTasks{}
if err := json.Unmarshal(bufData.Bytes(), &namespaces); err != nil {
return fmt.Errorf("could not read data: %s", err)
}
for _, n := range namespaces {
for _, l := range n.Lists {
if b, exists := storedFiles[l.BackgroundFileID]; exists {
bf, err := b.Open()
if err != nil {
return fmt.Errorf("could not open list background file %d for reading: %s", l.BackgroundFileID, err)
}
var buf bytes.Buffer
if _, err := buf.ReadFrom(bf); err != nil {
return fmt.Errorf("could not read list background file %d: %s", l.BackgroundFileID, err)
}
l.BackgroundInformation = &buf
}
for _, t := range l.Tasks {
for _, label := range t.Labels {
label.ID = 0
}
for _, comment := range t.Comments {
comment.ID = 0
}
for _, attachment := range t.Attachments {
af, err := storedFiles[attachment.File.ID].Open()
if err != nil {
return fmt.Errorf("could not open attachment %d for reading: %s", attachment.ID, err)
}
var buf bytes.Buffer
if _, err := buf.ReadFrom(af); err != nil {
return fmt.Errorf("could not read attachment %d: %s", attachment.ID, err)
}
attachment.ID = 0
attachment.File.ID = 0
attachment.File.FileContent = buf.Bytes()
}
}
}
}
err = migration.InsertFromStructure(namespaces, user)
if err != nil {
return fmt.Errorf("could not insert data: %s", err)
}
if filterFile == nil {
log.Debugf(logPrefix + "No filter file found")
return nil
}
///////
// Import filters
ff, err := filterFile.Open()
if err != nil {
return fmt.Errorf("could not open filters file: %s", err)
}
defer ff.Close()
var bufFilter bytes.Buffer
if _, err := bufFilter.ReadFrom(ff); err != nil {
return fmt.Errorf("could not read filters file: %s", err)
}
filters := []*models.SavedFilter{}
if err := json.Unmarshal(bufFilter.Bytes(), &filters); err != nil {
return fmt.Errorf("could not read filter data: %s", err)
}
log.Debugf(logPrefix+"Importing %d saved filters", len(filters))
s := db.NewSession()
defer s.Close()
for _, f := range filters {
f.ID = 0
err = f.Create(s, user)
if err != nil {
_ = s.Rollback()
return err
}
}
return s.Commit()
}

79
pkg/modules/migration/vikunja-file/vikunja_test.go Normal file
View File

@ -0,0 +1,79 @@
// Vikunja is a to-do list application to facilitate your life.
// Copyright 2018-2021 Vikunja and contributors. All rights reserved.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public Licensee as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public Licensee for more details.
//
// You should have received a copy of the GNU Affero General Public Licensee
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package vikunjafile
import (
"os"
"testing"
"code.vikunja.io/api/pkg/config"
"code.vikunja.io/api/pkg/db"
"code.vikunja.io/api/pkg/user"
"github.com/stretchr/testify/assert"
)
func TestVikunjaFileMigrator_Migrate(t *testing.T) {
db.LoadAndAssertFixtures(t)
m := &FileMigrator{}
u := &user.User{ID: 1}
f, err := os.Open(config.ServiceRootpath.GetString() + "/pkg/modules/migration/vikunja-file/export.zip")
if err != nil {
t.Fatalf("Could not open file: %s", err)
}
defer f.Close()
s, err := f.Stat()
if err != nil {
t.Fatalf("Could not stat file: %s", err)
}
err = m.Migrate(u, f, s.Size())
assert.NoError(t, err)
db.AssertExists(t, "namespaces", map[string]interface{}{
"title": "test",
"owner_id": u.ID,
}, false)
db.AssertExists(t, "lists", map[string]interface{}{
"title": "Test list",
"owner_id": u.ID,
}, false)
db.AssertExists(t, "lists", map[string]interface{}{
"title": "A list with a background",
"owner_id": u.ID,
}, false)
db.AssertExists(t, "tasks", map[string]interface{}{
"title": "Some other task",
"created_by_id": u.ID,
}, false)
db.AssertExists(t, "task_comments", map[string]interface{}{
"comment": "This is a comment",
"author_id": u.ID,
}, false)
db.AssertExists(t, "files", map[string]interface{}{
"name": "cristiano-mozzillo-v3d5uBB26yA-unsplash.jpg",
"created_by_id": u.ID,
}, false)
db.AssertExists(t, "labels", map[string]interface{}{
"title": "test",
"created_by_id": u.ID,
}, false)
db.AssertExists(t, "buckets", map[string]interface{}{
"title": "Test Bucket",
"created_by_id": u.ID,
}, false)
}