ami/tl-vikunja
1
0
Fork 0
Code Activity

fix(migration): make file migration work with new structure

Browse Source
This commit is contained in:
kolaente
2023-01-13 18:32:54 +01:00
parent 2f81791735
commit 4b55e2ce03
7 changed files with 176 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
pkg/modules/migration/vikunja-file/export.zip
View File

Binary file not shown.

BIN
pkg/modules/migration/vikunja-file/export_pre_0.21.0.zip Normal file
View File

Binary file not shown.

145
pkg/modules/migration/vikunja-file/vikunja.go
View File

@ -30,6 +30,8 @@ import (
"code.vikunja.io/api/pkg/models"
"code.vikunja.io/api/pkg/modules/migration"
"code.vikunja.io/api/pkg/user"
"github.com/hashicorp/go-version"
)
const logPrefix = "[Vikunja File Import] "
@ -71,6 +73,7 @@ func (v *FileMigrator) Migrate(user *user.User, file io.ReaderAt, size int64) er
var dataFile *zip.File
var filterFile *zip.File
var versionFile *zip.File
storedFiles := make(map[int64]*zip.File)
for _, f := range r.File {
if strings.HasPrefix(f.Name, "files/") {
@ -92,6 +95,10 @@ func (v *FileMigrator) Migrate(user *user.User, file io.ReaderAt, size int64) er
filterFile = f
log.Debugf(logPrefix + "Found a filter file")
}
if f.Name == "VERSION" {
versionFile = f
log.Debugf(logPrefix + "Found a version file")
}
}
if dataFile == nil {
@ -100,6 +107,31 @@ func (v *FileMigrator) Migrate(user *user.User, file io.ReaderAt, size int64) er
log.Debugf(logPrefix + "")
//////
// Check if we're able to import this dump
vf, err := versionFile.Open()
if err != nil {
return fmt.Errorf("could not open version file: %w", err)
}
var bufVersion bytes.Buffer
if _, err := bufVersion.ReadFrom(vf); err != nil {
return fmt.Errorf("could not read version file: %w", err)
}
dumpedVersion, err := version.NewVersion(bufVersion.String())
if err != nil {
return err
}
minVersion, err := version.NewVersion("0.20.1+61")
if err != nil {
return err
}
if dumpedVersion.LessThan(minVersion) {
return fmt.Errorf("export was created with an older version, need at least %s but the export needs at least %s", dumpedVersion, minVersion)
}
//////
// Import the bulk of Vikunja data
df, err := dataFile.Open()
@ -113,57 +145,19 @@ func (v *FileMigrator) Migrate(user *user.User, file io.ReaderAt, size int64) er
return fmt.Errorf("could not read data file: %w", err)
}
parent := []*models.ProjectWithTasksAndBuckets{}
if err := json.Unmarshal(bufData.Bytes(), &parent); err != nil {
projects := []*models.ProjectWithTasksAndBuckets{}
if err := json.Unmarshal(bufData.Bytes(), &projects); err != nil {
return fmt.Errorf("could not read data: %w", err)
}
for _, n := range parent {
for _, l := range n.ChildProjects {
if b, exists := storedFiles[l.BackgroundFileID]; exists {
bf, err := b.Open()
if err != nil {
return fmt.Errorf("could not open project background file %d for reading: %w", l.BackgroundFileID, err)
}
var buf bytes.Buffer
if _, err := buf.ReadFrom(bf); err != nil {
return fmt.Errorf("could not read project background file %d: %w", l.BackgroundFileID, err)
}
l.BackgroundInformation = &buf
}
for _, t := range l.Tasks {
for _, label := range t.Labels {
label.ID = 0
}
for _, comment := range t.Comments {
comment.ID = 0
}
for _, attachment := range t.Attachments {
attachmentFile, exists := storedFiles[attachment.File.ID]
if !exists {
log.Debugf(logPrefix+"Could not find attachment file %d for attachment %d", attachment.File.ID, attachment.ID)
continue
}
af, err := attachmentFile.Open()
if err != nil {
return fmt.Errorf("could not open attachment %d for reading: %w", attachment.ID, err)
}
var buf bytes.Buffer
if _, err := buf.ReadFrom(af); err != nil {
return fmt.Errorf("could not read attachment %d: %w", attachment.ID, err)
}
attachment.ID = 0
attachment.File.ID = 0
attachment.File.FileContent = buf.Bytes()
}
}
for _, p := range projects {
err = addDetailsToProjectAndChildren(p, storedFiles)
if err != nil {
return err
}
}
err = migration.InsertFromStructure(parent, user)
err = migration.InsertFromStructure(projects, user)
if err != nil {
return fmt.Errorf("could not insert data: %w", err)
}
@ -207,3 +201,64 @@ func (v *FileMigrator) Migrate(user *user.User, file io.ReaderAt, size int64) er
return s.Commit()
}
func addDetailsToProjectAndChildren(p *models.ProjectWithTasksAndBuckets, storedFiles map[int64]*zip.File) (err error) {
err = addDetailsToProject(p, storedFiles)
if err != nil {
return err
}
for _, cp := range p.ChildProjects {
err = addDetailsToProjectAndChildren(cp, storedFiles)
if err != nil {
return
}
}
return
}
func addDetailsToProject(l *models.ProjectWithTasksAndBuckets, storedFiles map[int64]*zip.File) (err error) {
if b, exists := storedFiles[l.BackgroundFileID]; exists {
bf, err := b.Open()
if err != nil {
return fmt.Errorf("could not open project background file %d for reading: %w", l.BackgroundFileID, err)
}
var buf bytes.Buffer
if _, err := buf.ReadFrom(bf); err != nil {
return fmt.Errorf("could not read project background file %d: %w", l.BackgroundFileID, err)
}
l.BackgroundInformation = &buf
}
for _, t := range l.Tasks {
for _, label := range t.Labels {
label.ID = 0
}
for _, comment := range t.Comments {
comment.ID = 0
}
for _, attachment := range t.Attachments {
attachmentFile, exists := storedFiles[attachment.File.ID]
if !exists {
log.Debugf(logPrefix+"Could not find attachment file %d for attachment %d", attachment.File.ID, attachment.ID)
continue
}
af, err := attachmentFile.Open()
if err != nil {
return fmt.Errorf("could not open attachment %d for reading: %w", attachment.ID, err)
}
var buf bytes.Buffer
if _, err := buf.ReadFrom(af); err != nil {
return fmt.Errorf("could not read attachment %d: %w", attachment.ID, err)
}
attachment.ID = 0
attachment.File.ID = 0
attachment.File.FileContent = buf.Bytes()
}
}
return
}

106
pkg/modules/migration/vikunja-file/vikunja_test.go
View File

@ -27,49 +27,71 @@ import (
)
func TestVikunjaFileMigrator_Migrate(t *testing.T) {
db.LoadAndAssertFixtures(t)
t.Run("migrate successfully", func(t *testing.T) {
db.LoadAndAssertFixtures(t)
m := &FileMigrator{}
u := &user.User{ID: 1}
m := &FileMigrator{}
u := &user.User{ID: 1}
f, err := os.Open(config.ServiceRootpath.GetString() + "/pkg/modules/migration/vikunja-file/export.zip")
if err != nil {
t.Fatalf("Could not open file: %s", err)
}
defer f.Close()
s, err := f.Stat()
if err != nil {
t.Fatalf("Could not stat file: %s", err)
}
f, err := os.Open(config.ServiceRootpath.GetString() + "/pkg/modules/migration/vikunja-file/export.zip")
if err != nil {
t.Fatalf("Could not open file: %s", err)
}
defer f.Close()
s, err := f.Stat()
if err != nil {
t.Fatalf("Could not stat file: %s", err)
}
err = m.Migrate(u, f, s.Size())
assert.NoError(t, err)
db.AssertExists(t, "projects", map[string]interface{}{
"title": "Test project",
"owner_id": u.ID,
}, false)
db.AssertExists(t, "projects", map[string]interface{}{
"title": "A project with a background",
"owner_id": u.ID,
}, false)
db.AssertExists(t, "tasks", map[string]interface{}{
"title": "Some other task",
"created_by_id": u.ID,
}, false)
db.AssertExists(t, "task_comments", map[string]interface{}{
"comment": "This is a comment",
"author_id": u.ID,
}, false)
db.AssertExists(t, "files", map[string]interface{}{
"name": "cristiano-mozzillo-v3d5uBB26yA-unsplash.jpg",
"created_by_id": u.ID,
}, false)
db.AssertExists(t, "labels", map[string]interface{}{
"title": "test",
"created_by_id": u.ID,
}, false)
db.AssertExists(t, "buckets", map[string]interface{}{
"title": "Test Bucket",
"created_by_id": u.ID,
}, false)
err = m.Migrate(u, f, s.Size())
assert.NoError(t, err)
db.AssertExists(t, "projects", map[string]interface{}{
"title": "test project",
"owner_id": u.ID,
}, false)
db.AssertExists(t, "projects", map[string]interface{}{
"title": "Inbox",
"owner_id": u.ID,
}, false)
db.AssertExists(t, "tasks", map[string]interface{}{
"title": "some other task",
"created_by_id": u.ID,
}, false)
db.AssertExists(t, "task_comments", map[string]interface{}{
"comment": "This is a comment",
"author_id": u.ID,
}, false)
db.AssertExists(t, "files", map[string]interface{}{
"name": "grant-whitty-546453-unsplash.jpg",
"created_by_id": u.ID,
}, false)
db.AssertExists(t, "labels", map[string]interface{}{
"title": "test",
"created_by_id": u.ID,
}, false)
db.AssertExists(t, "buckets", map[string]interface{}{
"title": "Test Bucket",
"created_by_id": u.ID,
}, false)
})
t.Run("should not accept an old import", func(t *testing.T) {
db.LoadAndAssertFixtures(t)
m := &FileMigrator{}
u := &user.User{ID: 1}
f, err := os.Open(config.ServiceRootpath.GetString() + "/pkg/modules/migration/vikunja-file/export_pre_0.21.0.zip")
if err != nil {
t.Fatalf("Could not open file: %s", err)
}
defer f.Close()
s, err := f.Stat()
if err != nil {
t.Fatalf("Could not stat file: %s", err)
}
err = m.Migrate(u, f, s.Size())
assert.Error(t, err)
assert.ErrorContainsf(t, err, "export was created with an older version", "Invalid error message")
})
}