ami/tl-vikunja
1
0
Fork 0
Code Activity

Migrate all timestamps to real iso dates (#594)

Browse Source 
Fix query param name

Add option to include null results when filtering

Always set db time to gmt

Fix null filter

Fix timezone setting for todoist parsing

Fix timezone setting for wunderlist parsing

Fix import

Fix caldav reminder parsing

Use timezone from config

Add error and test for invalid filter values

Fix integration tests

Remove task collection date hack

Fix task filter

Fix lint

Fix tests and fixtures for date timezone stuff

Properly set timezone

Change fixtures time zone to gmt

Set db timezone

Set created and updated timestamps for all fixtures

Fix lint

Fix test fixtures

Fix misspell

Fix test fixtures

Partially fix tests

Remove timeutil package

Remove adding _unix suffix hack

Remove _unix suffix

Move all timeutil.TimeStamp to time.Time

Remove all Unix suffixes in field names

Add better error messages when running migrations

Make sure to not migrate 0 unix timestamps to 1970 iso dates

Add migration script for sqlite

Add converting sqlite values

Convert 0 unix timestamps to null in postgres

Convert 0 to null in timestamps

Automatically rename _unix suffix

Add all tables and columns for migration

Fix sql migration query for mysql

Fail with an error if trying to use an unsupported dbms

Co-authored-by: kolaente <k@knt.li>
Reviewed-on: https://kolaente.dev/vikunja/api/pulls/594
This commit is contained in:
konrad
2020-06-27 17:04:01 +00:00
parent e17cac854a
commit 08205008e7
83 changed files with 2208 additions and 1257 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
pkg/modules/migration/migration_status.go
View File

@ -17,16 +17,16 @@
package migration
import (
"code.vikunja.io/api/pkg/timeutil"
"code.vikunja.io/api/pkg/user"
"time"
)
// Status represents this migration status
type Status struct {
ID int64 `xorm:"int(11) autoincr not null unique pk" json:"id"`
UserID int64 `xorm:"int(11) not null" json:"-"`
MigratorName string `xorm:"varchar(255)" json:"migrator_name"`
Created timeutil.TimeStamp `xorm:"created not null 'created_unix'" json:"time_unix"`
ID int64 `xorm:"int(11) autoincr not null unique pk" json:"id"`
UserID int64 `xorm:"int(11) not null" json:"-"`
MigratorName string `xorm:"varchar(255)" json:"migrator_name"`
Created time.Time `xorm:"created not null 'created'" json:"time"`
}
// TableName holds the table name for the migration status table

20
pkg/modules/migration/todoist/todoist.go
View File

@ -23,7 +23,6 @@ import (
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/models"
"code.vikunja.io/api/pkg/modules/migration"
"code.vikunja.io/api/pkg/timeutil"
"code.vikunja.io/api/pkg/user"
"code.vikunja.io/api/pkg/utils"
"encoding/json"
@ -267,14 +266,14 @@ func convertTodoistToVikunja(sync *sync) (fullVikunjaHierachie []*models.Namespa
for _, i := range sync.Items {
task := &models.Task{
Title: i.Content,
Created: timeutil.FromTime(i.DateAdded),
Created: i.DateAdded.In(config.GetTimeZone()),
Done: i.Checked == 1,
}
// Only try to parse the task done at date if the task is actually done
// Sometimes weired things happen if we try to parse nil dates.
if task.Done {
task.DoneAt = timeutil.FromTime(i.DateCompleted)
task.DoneAt = i.DateCompleted.In(config.GetTimeZone())
}
// Todoist priorities only range from 1 (lowest) and max 4 (highest), so we need to make slight adjustments
@ -288,7 +287,7 @@ func convertTodoistToVikunja(sync *sync) (fullVikunjaHierachie []*models.Namespa
if err != nil {
return nil, err
}
task.DueDate = timeutil.FromTime(dueDate)
task.DueDate = dueDate.In(config.GetTimeZone())
}
// Put all labels together from earlier
@ -341,17 +340,16 @@ func convertTodoistToVikunja(sync *sync) (fullVikunjaHierachie []*models.Namespa
tasks[n.ItemID].Attachments = append(tasks[n.ItemID].Attachments, &models.TaskAttachment{
File: &files.File{
Name: n.FileAttachment.FileName,
Mime: n.FileAttachment.FileType,
Size: uint64(n.FileAttachment.FileSize),
Created: n.Posted,
CreatedUnix: timeutil.FromTime(n.Posted),
Name: n.FileAttachment.FileName,
Mime: n.FileAttachment.FileType,
Size: uint64(n.FileAttachment.FileSize),
Created: n.Posted,
// We directly pass the file contents here to have a way to link the attachment to the file later.
// Because we don't have an ID for our task at this point of the migration, we cannot just throw all
// attachments in a slice and do the work of downloading and properly storing them later.
FileContent: buf.Bytes(),
},
Created: timeutil.FromTime(n.Posted),
Created: n.Posted,
})
}
@ -375,7 +373,7 @@ func convertTodoistToVikunja(sync *sync) (fullVikunjaHierachie []*models.Namespa
return nil, err
}
tasks[r.ItemID].Reminders = append(tasks[r.ItemID].Reminders, timeutil.FromTime(date))
tasks[r.ItemID].Reminders = append(tasks[r.ItemID].Reminders, date.In(config.GetTimeZone()))
}
return []*models.NamespaceWithLists{

116
pkg/modules/migration/todoist/todoist_test.go
View File

@ -20,7 +20,6 @@ import (
"code.vikunja.io/api/pkg/config"
"code.vikunja.io/api/pkg/files"
"code.vikunja.io/api/pkg/models"
"code.vikunja.io/api/pkg/timeutil"
"github.com/stretchr/testify/assert"
"gopkg.in/d4l3k/messagediff.v1"
"io/ioutil"
@ -35,12 +34,16 @@ func TestConvertTodoistToVikunja(t *testing.T) {
time1, err := time.Parse(time.RFC3339Nano, "2014-09-26T08:25:05Z")
assert.NoError(t, err)
time1 = time1.In(config.GetTimeZone())
time3, err := time.Parse(time.RFC3339Nano, "2014-10-21T08:25:05Z")
assert.NoError(t, err)
time3 = time3.In(config.GetTimeZone())
dueTime, err := time.Parse(time.RFC3339Nano, "2020-05-31T00:00:00Z")
assert.NoError(t, err)
nilTime, err := time.Parse(time.RFC3339Nano, "1970-01-01T00:00:00Z")
dueTime = dueTime.In(config.GetTimeZone())
nilTime, err := time.Parse(time.RFC3339Nano, "0001-01-01T00:00:00Z")
assert.NoError(t, err)
//nilTime = nilTime.In(config.GetTimeZone())
exampleFile, err := ioutil.ReadFile(config.ServiceRootpath.GetString() + "/pkg/modules/migration/wunderlist/testimage.jpg")
assert.NoError(t, err)
@ -344,68 +347,68 @@ func TestConvertTodoistToVikunja(t *testing.T) {
Title: "Task400000000",
Description: "Lorem Ipsum dolor sit amet",
Done: false,
Created: timeutil.FromTime(time1),
Reminders: []timeutil.TimeStamp{
timeutil.FromTime(time.Date(2020, time.June, 15, 0, 0, 0, 0, time.UTC)),
timeutil.FromTime(time.Date(2020, time.June, 16, 0, 0, 0, 0, time.UTC)),
Created: time1,
Reminders: []time.Time{
time.Date(2020, time.June, 15, 0, 0, 0, 0, time.UTC).In(config.GetTimeZone()),
time.Date(2020, time.June, 16, 0, 0, 0, 0, time.UTC).In(config.GetTimeZone()),
},
},
{
Title: "Task400000001",
Description: "Lorem Ipsum dolor sit amet",
Done: false,
Created: timeutil.FromTime(time1),
Created: time1,
},
{
Title: "Task400000002",
Done: false,
Created: timeutil.FromTime(time1),
Reminders: []timeutil.TimeStamp{
timeutil.FromTime(time.Date(2020, time.July, 15, 0, 0, 0, 0, time.UTC)),
Created: time1,
Reminders: []time.Time{
time.Date(2020, time.July, 15, 0, 0, 0, 0, time.UTC).In(config.GetTimeZone()),
},
},
{
Title: "Task400000003",
Description: "Lorem Ipsum dolor sit amet",
Done: true,
DueDate: timeutil.FromTime(dueTime),
Created: timeutil.FromTime(time1),
DoneAt: timeutil.FromTime(time3),
DueDate: dueTime,
Created: time1,
DoneAt: time3,
Labels: vikunjaLabels,
Reminders: []timeutil.TimeStamp{
timeutil.FromTime(time.Date(2020, time.June, 15, 0, 0, 0, 0, time.UTC)),
Reminders: []time.Time{
time.Date(2020, time.June, 15, 0, 0, 0, 0, time.UTC).In(config.GetTimeZone()),
},
},
{
Title: "Task400000004",
Done: false,
Created: timeutil.FromTime(time1),
Created: time1,
Labels: vikunjaLabels,
},
{
Title: "Task400000005",
Done: true,
DueDate: timeutil.FromTime(dueTime),
Created: timeutil.FromTime(time1),
DoneAt: timeutil.FromTime(time3),
Reminders: []timeutil.TimeStamp{
timeutil.FromTime(time.Date(2020, time.June, 15, 0, 0, 0, 0, time.UTC)),
DueDate: dueTime,
Created: time1,
DoneAt: time3,
Reminders: []time.Time{
time.Date(2020, time.June, 15, 0, 0, 0, 0, time.UTC).In(config.GetTimeZone()),
},
},
{
Title: "Task400000006",
Done: true,
DueDate: timeutil.FromTime(dueTime),
Created: timeutil.FromTime(time1),
DoneAt: timeutil.FromTime(time3),
DueDate: dueTime,
Created: time1,
DoneAt: time3,
RelatedTasks: map[models.RelationKind][]*models.Task{
models.RelationKindSubtask: {
{
Title: "Task with parent",
Done: false,
Priority: 2,
Created: timeutil.FromTime(time1),
DoneAt: timeutil.FromTime(nilTime),
Created: time1,
DoneAt: nilTime,
},
},
},
@ -414,34 +417,34 @@ func TestConvertTodoistToVikunja(t *testing.T) {
Title: "Task with parent",
Done: false,
Priority: 2,
Created: timeutil.FromTime(time1),
DoneAt: timeutil.FromTime(nilTime),
Created: time1,
DoneAt: nilTime,
},
{
Title: "Task400000106",
Done: true,
DueDate: timeutil.FromTime(dueTime),
Created: timeutil.FromTime(time1),
DoneAt: timeutil.FromTime(time3),
DueDate: dueTime,
Created: time1,
DoneAt: time3,
Labels: vikunjaLabels,
},
{
Title: "Task400000107",
Done: true,
Created: timeutil.FromTime(time1),
DoneAt: timeutil.FromTime(time3),
Created: time1,
DoneAt: time3,
},
{
Title: "Task400000108",
Done: true,
Created: timeutil.FromTime(time1),
DoneAt: timeutil.FromTime(time3),
Created: time1,
DoneAt: time3,
},
{
Title: "Task400000109",
Done: true,
Created: timeutil.FromTime(time1),
DoneAt: timeutil.FromTime(time3),
Created: time1,
DoneAt: time3,
},
},
},
@ -453,35 +456,35 @@ func TestConvertTodoistToVikunja(t *testing.T) {
{
Title: "Task400000007",
Done: false,
DueDate: timeutil.FromTime(dueTime),
Created: timeutil.FromTime(time1),
DueDate: dueTime,
Created: time1,
},
{
Title: "Task400000008",
Done: false,
DueDate: timeutil.FromTime(dueTime),
Created: timeutil.FromTime(time1),
DueDate: dueTime,
Created: time1,
},
{
Title: "Task400000009",
Done: false,
Created: timeutil.FromTime(time1),
Reminders: []timeutil.TimeStamp{
timeutil.FromTime(time.Date(2020, time.June, 15, 0, 0, 0, 0, time.UTC)),
Created: time1,
Reminders: []time.Time{
time.Date(2020, time.June, 15, 0, 0, 0, 0, time.UTC).In(config.GetTimeZone()),
},
},
{
Title: "Task400000010",
Description: "Lorem Ipsum dolor sit amet",
Done: true,
Created: timeutil.FromTime(time1),
DoneAt: timeutil.FromTime(time3),
Created: time1,
DoneAt: time3,
},
{
Title: "Task400000101",
Description: "Lorem Ipsum dolor sit amet",
Done: false,
Created: timeutil.FromTime(time1),
Created: time1,
Attachments: []*models.TaskAttachment{
{
File: &files.File{
@ -489,37 +492,36 @@ func TestConvertTodoistToVikunja(t *testing.T) {
Mime: "text/plain",
Size: 12345,
Created: time1,
CreatedUnix: timeutil.FromTime(time1),
FileContent: exampleFile,
},
Created: timeutil.FromTime(time1),
Created: time1,
},
},
},
{
Title: "Task400000102",
Done: false,
DueDate: timeutil.FromTime(dueTime),
Created: timeutil.FromTime(time1),
DueDate: dueTime,
Created: time1,
Labels: vikunjaLabels,
},
{
Title: "Task400000103",
Done: false,
Created: timeutil.FromTime(time1),
Created: time1,
Labels: vikunjaLabels,
},
{
Title: "Task400000104",
Done: false,
Created: timeutil.FromTime(time1),
Created: time1,
Labels: vikunjaLabels,
},
{
Title: "Task400000105",
Done: false,
DueDate: timeutil.FromTime(dueTime),
Created: timeutil.FromTime(time1),
DueDate: dueTime,
Created: time1,
Labels: vikunjaLabels,
},
},
@ -532,8 +534,8 @@ func TestConvertTodoistToVikunja(t *testing.T) {
{
Title: "Task400000111",
Done: true,
Created: timeutil.FromTime(time1),
DoneAt: timeutil.FromTime(time3),
Created: time1,
DoneAt: time3,
},
},
},

26
pkg/modules/migration/wunderlist/wunderlist.go
View File

@ -23,7 +23,6 @@ import (
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/models"
"code.vikunja.io/api/pkg/modules/migration"
"code.vikunja.io/api/pkg/timeutil"
"code.vikunja.io/api/pkg/user"
"code.vikunja.io/api/pkg/utils"
"encoding/json"
@ -145,7 +144,7 @@ func convertListForFolder(listID int, list *list, content *wunderlistContents) (
l := &models.List{
Title: list.Title,
Created: timeutil.FromTime(list.CreatedAt),
Created: list.CreatedAt,
}
// Find all tasks belonging to this list and put them in
@ -153,13 +152,13 @@ func convertListForFolder(listID int, list *list, content *wunderlistContents) (
if t.ListID == listID {
newTask := &models.Task{
Title: t.Title,
Created: timeutil.FromTime(t.CreatedAt),
Created: t.CreatedAt,
Done: t.Completed,
}
// Set Done At
if newTask.Done {
newTask.DoneAt = timeutil.FromTime(t.CompletedAt)
newTask.DoneAt = t.CompletedAt.In(config.GetTimeZone())
}
// Parse the due date
@ -168,7 +167,7 @@ func convertListForFolder(listID int, list *list, content *wunderlistContents) (
if err != nil {
return nil, err
}
newTask.DueDate = timeutil.FromTime(dueDate)
newTask.DueDate = dueDate.In(config.GetTimeZone())
}
// Find related notes
@ -195,17 +194,16 @@ func convertListForFolder(listID int, list *list, content *wunderlistContents) (
newTask.Attachments = append(newTask.Attachments, &models.TaskAttachment{
File: &files.File{
Name: f.FileName,
Mime: f.ContentType,
Size: uint64(f.FileSize),
Created: f.CreatedAt,
CreatedUnix: timeutil.FromTime(f.CreatedAt),
Name: f.FileName,
Mime: f.ContentType,
Size: uint64(f.FileSize),
Created: f.CreatedAt,
// We directly pass the file contents here to have a way to link the attachment to the file later.
// Because we don't have an ID for our task at this point of the migration, we cannot just throw all
// attachments in a slice and do the work of downloading and properly storing them later.
FileContent: buf.Bytes(),
},
Created: timeutil.FromTime(f.CreatedAt),
Created: f.CreatedAt,
})
}
}
@ -225,7 +223,7 @@ func convertListForFolder(listID int, list *list, content *wunderlistContents) (
// Reminders
for _, r := range content.reminders {
if r.TaskID == t.ID {
newTask.Reminders = append(newTask.Reminders, timeutil.FromTime(r.Date))
newTask.Reminders = append(newTask.Reminders, r.Date.In(config.GetTimeZone()))
}
}
@ -248,8 +246,8 @@ func convertWunderlistToVikunja(content *wunderlistContents) (fullVikunjaHierach
namespace := &models.NamespaceWithLists{
Namespace: models.Namespace{
Title: folder.Title,
Created: timeutil.FromTime(folder.CreatedAt),
Updated: timeutil.FromTime(folder.UpdatedAt),
Created: folder.CreatedAt,
Updated: folder.UpdatedAt,
},
}

80
pkg/modules/migration/wunderlist/wunderlist_test.go
View File

@ -20,7 +20,6 @@ import (
"code.vikunja.io/api/pkg/config"
"code.vikunja.io/api/pkg/files"
"code.vikunja.io/api/pkg/models"
"code.vikunja.io/api/pkg/timeutil"
"github.com/stretchr/testify/assert"
"gopkg.in/d4l3k/messagediff.v1"
"io/ioutil"
@ -35,12 +34,16 @@ func TestWunderlistParsing(t *testing.T) {
time1, err := time.Parse(time.RFC3339Nano, "2013-08-30T08:29:46.203Z")
assert.NoError(t, err)
time1 = time1.In(config.GetTimeZone())
time2, err := time.Parse(time.RFC3339Nano, "2013-08-30T08:36:13.273Z")
assert.NoError(t, err)
time2 = time2.In(config.GetTimeZone())
time3, err := time.Parse(time.RFC3339Nano, "2013-09-05T08:36:13.273Z")
assert.NoError(t, err)
time3 = time3.In(config.GetTimeZone())
time4, err := time.Parse(time.RFC3339Nano, "2013-08-02T11:58:55Z")
assert.NoError(t, err)
time4 = time4.In(config.GetTimeZone())
exampleFile, err := ioutil.ReadFile(config.ServiceRootpath.GetString() + "/pkg/modules/migration/wunderlist/testimage.jpg")
assert.NoError(t, err)
@ -51,6 +54,7 @@ func TestWunderlistParsing(t *testing.T) {
if done {
completedAt = time1
}
completedAt = completedAt.In(config.GetTimeZone())
return &task{
ID: id,
AssigneeID: 123,
@ -193,18 +197,18 @@ func TestWunderlistParsing(t *testing.T) {
{
Namespace: models.Namespace{
Title: "Lorem Ipsum",
Created: timeutil.FromTime(time1),
Updated: timeutil.FromTime(time2),
Created: time1,
Updated: time2,
},
Lists: []*models.List{
{
Created: timeutil.FromTime(time1),
Created: time1,
Title: "Lorem1",
Tasks: []*models.Task{
{
Title: "Ipsum1",
DueDate: 1378339200,
Created: timeutil.FromTime(time1),
DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()),
Created: time1,
Description: "Lorem Ipsum dolor sit amet",
Attachments: []*models.TaskAttachment{
{
@ -213,18 +217,17 @@ func TestWunderlistParsing(t *testing.T) {
Mime: "text/plain",
Size: 12345,
Created: time2,
CreatedUnix: timeutil.FromTime(time2),
FileContent: exampleFile,
},
Created: timeutil.FromTime(time2),
Created: time2,
},
},
Reminders: []timeutil.TimeStamp{timeutil.FromTime(time4)},
Reminders: []time.Time{time4},
},
{
Title: "Ipsum2",
DueDate: 1378339200,
Created: timeutil.FromTime(time1),
DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()),
Created: time1,
Description: "Lorem Ipsum dolor sit amet",
RelatedTasks: map[models.RelationKind][]*models.Task{
models.RelationKindSubtask: {
@ -240,15 +243,15 @@ func TestWunderlistParsing(t *testing.T) {
},
},
{
Created: timeutil.FromTime(time1),
Created: time1,
Title: "Lorem2",
Tasks: []*models.Task{
{
Title: "Ipsum3",
Done: true,
DoneAt: timeutil.FromTime(time1),
DueDate: 1378339200,
Created: timeutil.FromTime(time1),
DoneAt: time1,
DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()),
Created: time1,
Description: "Lorem Ipsum dolor sit amet",
Attachments: []*models.TaskAttachment{
{
@ -257,18 +260,17 @@ func TestWunderlistParsing(t *testing.T) {
Mime: "text/plain",
Size: 12345,
Created: time3,
CreatedUnix: timeutil.FromTime(time3),
FileContent: exampleFile,
},
Created: timeutil.FromTime(time3),
Created: time3,
},
},
},
{
Title: "Ipsum4",
DueDate: 1378339200,
Created: timeutil.FromTime(time1),
Reminders: []timeutil.TimeStamp{timeutil.FromTime(time3)},
DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()),
Created: time1,
Reminders: []time.Time{time3},
RelatedTasks: map[models.RelationKind][]*models.Task{
models.RelationKindSubtask: {
{
@ -280,52 +282,52 @@ func TestWunderlistParsing(t *testing.T) {
},
},
{
Created: timeutil.FromTime(time1),
Created: time1,
Title: "Lorem3",
Tasks: []*models.Task{
{
Title: "Ipsum5",
DueDate: 1378339200,
Created: timeutil.FromTime(time1),
DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()),
Created: time1,
},
{
Title: "Ipsum6",
DueDate: 1378339200,
Created: timeutil.FromTime(time1),
DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()),
Created: time1,
Done: true,
DoneAt: timeutil.FromTime(time1),
DoneAt: time1,
},
{
Title: "Ipsum7",
DueDate: 1378339200,
Created: timeutil.FromTime(time1),
DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()),
Created: time1,
Done: true,
DoneAt: timeutil.FromTime(time1),
DoneAt: time1,
},
{
Title: "Ipsum8",
DueDate: 1378339200,
Created: timeutil.FromTime(time1),
DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()),
Created: time1,
},
},
},
{
Created: timeutil.FromTime(time1),
Created: time1,
Title: "Lorem4",
Tasks: []*models.Task{
{
Title: "Ipsum9",
DueDate: 1378339200,
Created: timeutil.FromTime(time1),
DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()),
Created: time1,
Done: true,
DoneAt: timeutil.FromTime(time1),
DoneAt: time1,
},
{
Title: "Ipsum10",
DueDate: 1378339200,
Created: timeutil.FromTime(time1),
DueDate: time.Unix(1378339200, 0).In(config.GetTimeZone()),
Created: time1,
Done: true,
DoneAt: timeutil.FromTime(time1),
DoneAt: time1,
},
},
},
@ -337,7 +339,7 @@ func TestWunderlistParsing(t *testing.T) {
},
Lists: []*models.List{
{
Created: timeutil.FromTime(time4),
Created: time4,
Title: "List without a namespace",
},
},