DB Migrations (#67)

pkg/cmd/migrate.go

@@ -0,0 +1,59 @@
+//   Vikunja is a todo-list application to facilitate your life.
+//   Copyright 2019 Vikunja and contributors. All rights reserved.
+//
+//   This program is free software: you can redistribute it and/or modify
+//   it under the terms of the GNU General Public License as published by
+//   the Free Software Foundation, either version 3 of the License, or
+//   (at your option) any later version.
+//
+//   This program is distributed in the hope that it will be useful,
+//   but WITHOUT ANY WARRANTY; without even the implied warranty of
+//   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+//   GNU General Public License for more details.
+//
+//   You should have received a copy of the GNU General Public License
+//   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"code.vikunja.io/api/pkg/migration"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	migrateCmd.AddCommand(migrateListCmd)
+	migrationRollbackCmd.Flags().StringVarP(&rollbackUntilFlag, "name", "n", "", "The id of the migration you want to roll back until.")
+	migrationRollbackCmd.MarkFlagRequired("name")
+	migrateCmd.AddCommand(migrationRollbackCmd)
+	rootCmd.AddCommand(migrateCmd)
+}
+
+// TODO: add args to run migrations up or down, until a certain point etc
+// Rollback until
+// list -> Essentially just show the table, maybe with an extra column if the migration did run or not
+var migrateCmd = &cobra.Command{
+	Use:   "migrate",
+	Short: "Run all database migrations which didn't already run.",
+	Run: func(cmd *cobra.Command, args []string) {
+		migration.Migrate(nil)
+	},
+}
+
+var migrateListCmd = &cobra.Command{
+	Use:   "list",
+	Short: "Show a list with all database migrations.",
+	Run: func(cmd *cobra.Command, args []string) {
+		migration.ListMigrations()
+	},
+}
+
+var rollbackUntilFlag string
+
+var migrationRollbackCmd = &cobra.Command{
+	Use:   "rollback",
+	Short: "Roll migrations back until a certain point.",
+	Run: func(cmd *cobra.Command, args []string) {
+		migration.Rollback(rollbackUntilFlag)
+	},
+}

pkg/cmd/web.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"code.vikunja.io/api/pkg/log"
 	"code.vikunja.io/api/pkg/mail"
+	"code.vikunja.io/api/pkg/migration"
 	"code.vikunja.io/api/pkg/models"
 	"code.vikunja.io/api/pkg/routes"
 	"code.vikunja.io/api/pkg/swagger"
@@ -43,6 +44,9 @@ var webCmd = &cobra.Command{
 		// Set logger
 		log.InitLogger()
 
+		// Run the migrations
+		migration.Migrate(nil)
+
 		// Set Engine
 		err := models.SetEngine()
 		if err != nil {

pkg/db/db.go

@@ -0,0 +1,78 @@
+//  Vikunja is a todo-list application to facilitate your life.
+//  Copyright 2019 Vikunja and contributors. All rights reserved.
+//
+//  This program is free software: you can redistribute it and/or modify
+//  it under the terms of the GNU General Public License as published by
+//  the Free Software Foundation, either version 3 of the License, or
+//  (at your option) any later version.
+//
+//  This program is distributed in the hope that it will be useful,
+//  but WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+//  GNU General Public License for more details.
+//
+//  You should have received a copy of the GNU General Public License
+//  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package db
+
+import (
+	"code.vikunja.io/api/pkg/config"
+	"code.vikunja.io/api/pkg/log"
+	"fmt"
+	"github.com/go-xorm/core"
+	"github.com/go-xorm/xorm"
+	"github.com/spf13/viper"
+
+	_ "github.com/go-sql-driver/mysql" // Because.
+	_ "github.com/mattn/go-sqlite3"    // Because.
+)
+
+// CreateDBEngine initializes a db engine from the config
+func CreateDBEngine() (engine *xorm.Engine, err error) {
+	// If the database type is not set, this likely means we need to initialize the config first
+	if viper.GetString("database.type") == "" {
+		config.InitConfig()
+	}
+
+	// Use Mysql if set
+	if viper.GetString("database.type") == "mysql" {
+		engine, err = initMysqlEngine()
+		if err != nil {
+			return
+		}
+	} else {
+		// Otherwise use sqlite
+		engine, err = initSqliteEngine()
+		if err != nil {
+			return
+		}
+	}
+
+	engine.SetMapper(core.GonicMapper{})
+	engine.ShowSQL(viper.GetString("log.database") != "off")
+	engine.SetLogger(xorm.NewSimpleLogger(log.GetLogWriter("database")))
+
+	return
+}
+
+func initMysqlEngine() (engine *xorm.Engine, err error) {
+	connStr := fmt.Sprintf(
+		"%s:%s@tcp(%s)/%s?charset=utf8&parseTime=true",
+		viper.GetString("database.user"),
+		viper.GetString("database.password"),
+		viper.GetString("database.host"),
+		viper.GetString("database.database"))
+	engine, err = xorm.NewEngine("mysql", connStr)
+	engine.SetMaxOpenConns(viper.GetInt("database.openconnections"))
+	return
+}
+
+func initSqliteEngine() (engine *xorm.Engine, err error) {
+	path := viper.GetString("database.path")
+	if path == "" {
+		path = "./db.db"
+	}
+
+	return xorm.NewEngine("sqlite3", path)
+}

pkg/migration/20190324205606.go

@@ -0,0 +1,44 @@
+//   Vikunja is a todo-list application to facilitate your life.
+//   Copyright 2019 Vikunja and contributors. All rights reserved.
+//
+//   This program is free software: you can redistribute it and/or modify
+//   it under the terms of the GNU General Public License as published by
+//   the Free Software Foundation, either version 3 of the License, or
+//   (at your option) any later version.
+//
+//   This program is distributed in the hope that it will be useful,
+//   but WITHOUT ANY WARRANTY; without even the implied warranty of
+//   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+//   GNU General Public License for more details.
+//
+//   You should have received a copy of the GNU General Public License
+//   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package migration
+
+import (
+	"github.com/go-xorm/xorm"
+	"src.techknowlogick.com/xormigrate"
+)
+
+// Used for rollback
+type tasksReminderDateMigration20190324205606 struct {
+	ReminderUnix int64 `xorm:"int(11) INDEX"`
+}
+
+func (tasksReminderDateMigration20190324205606) TableName() string {
+	return "tasks"
+}
+
+func init() {
+	migrations = append(migrations, &xormigrate.Migration{
+		ID:          "20190324205606",
+		Description: "Remove reminders_unix from tasks",
+		Migrate: func(tx *xorm.Engine) error {
+			return dropTableColum(tx, "tasks", "reminders_unix")
+		},
+		Rollback: func(tx *xorm.Engine) error {
+			return tx.Sync2(tasksReminderDateMigration20190324205606{})
+		},
+	})
+}

pkg/migration/20190328074430.go

@@ -0,0 +1,44 @@
+//  Vikunja is a todo-list application to facilitate your life.
+//  Copyright 2019 Vikunja and contributors. All rights reserved.
+//
+//  This program is free software: you can redistribute it and/or modify
+//  it under the terms of the GNU General Public License as published by
+//  the Free Software Foundation, either version 3 of the License, or
+//  (at your option) any later version.
+//
+//  This program is distributed in the hope that it will be useful,
+//  but WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+//  GNU General Public License for more details.
+//
+//  You should have received a copy of the GNU General Public License
+//  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package migration
+
+import (
+	"github.com/go-xorm/xorm"
+	"src.techknowlogick.com/xormigrate"
+)
+
+// Used for rollback
+type teamMembersMigration20190328074430 struct {
+	Updated int64 `xorm:"updated"`
+}
+
+func (teamMembersMigration20190328074430) TableName() string {
+	return "team_members"
+}
+
+func init() {
+	migrations = append(migrations, &xormigrate.Migration{
+		ID:          "20190328074430",
+		Description: "Remove updated from team_members",
+		Migrate: func(tx *xorm.Engine) error {
+			return dropTableColum(tx, "team_members", "updated")
+		},
+		Rollback: func(tx *xorm.Engine) error {
+			return tx.Sync2(teamMembersMigration20190328074430{})
+		},
+	})
+}

pkg/migration/migration.go

@@ -0,0 +1,124 @@
+//   Vikunja is a todo-list application to facilitate your life.
+//   Copyright 2019 Vikunja and contributors. All rights reserved.
+//
+//   This program is free software: you can redistribute it and/or modify
+//   it under the terms of the GNU General Public License as published by
+//   the Free Software Foundation, either version 3 of the License, or
+//   (at your option) any later version.
+//
+//   This program is distributed in the hope that it will be useful,
+//   but WITHOUT ANY WARRANTY; without even the implied warranty of
+//   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+//   GNU General Public License for more details.
+//
+//   You should have received a copy of the GNU General Public License
+//   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package migration
+
+import (
+	"code.vikunja.io/api/pkg/db"
+	"code.vikunja.io/api/pkg/log"
+	"code.vikunja.io/api/pkg/models"
+	"github.com/go-xorm/xorm"
+	"github.com/olekukonko/tablewriter"
+	"github.com/spf13/viper"
+	"os"
+	"sort"
+	"src.techknowlogick.com/xormigrate"
+)
+
+// You can get the id string for new migrations by running `date +%Y%m%d%H%M%S` on a unix system.
+
+var migrations []*xormigrate.Migration
+
+// A helper function because we need a migration in various places which we can't really solve with an init() function.
+func initMigration(x *xorm.Engine) *xormigrate.Xormigrate {
+	// Get our own xorm engine if we don't have one
+	if x == nil {
+		var err error
+		x, err = db.CreateDBEngine()
+		if err != nil {
+			log.Log.Criticalf("Could not connect to db: %v", err.Error())
+			return nil
+		}
+	}
+
+	// Because init() does not guarantee the order in which these are added to the slice,
+	// we need to sort them to ensure that they are in order
+	sort.Slice(migrations, func(i, j int) bool {
+		return migrations[i].ID < migrations[j].ID
+	})
+
+	m := xormigrate.New(x, migrations)
+	m.NewLogger(log.GetLogWriter("database"))
+	m.InitSchema(initSchema)
+	return m
+}
+
+// Migrate runs all migrations
+func Migrate(x *xorm.Engine) {
+	m := initMigration(x)
+	err := m.Migrate()
+	if err != nil {
+		log.Log.Fatalf("Migration failed: %v", err)
+	}
+	log.Log.Info("Ran all migrations successfully.")
+}
+
+// ListMigrations pretty-prints a list with all migrations.
+func ListMigrations() {
+	x, err := db.CreateDBEngine()
+	if err != nil {
+		log.Log.Fatalf("Could not connect to db: %v", err.Error())
+	}
+	ms := []*xormigrate.Migration{}
+	err = x.Find(&ms)
+	if err != nil {
+		log.Log.Fatalf("Error getting migration table: %v", err.Error())
+	}
+
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetHeader([]string{"ID", "Description"})
+	table.SetAlignment(tablewriter.ALIGN_LEFT)
+	table.SetHeaderColor(tablewriter.Colors{tablewriter.Bold, tablewriter.BgGreenColor},
+		tablewriter.Colors{tablewriter.Bold, tablewriter.BgGreenColor})
+
+	for _, m := range ms {
+		table.Append([]string{m.ID, m.Description})
+	}
+	table.Render()
+}
+
+// Rollback rolls back all migrations until a certain point.
+func Rollback(migrationID string) {
+	m := initMigration(nil)
+	err := m.RollbackTo(migrationID)
+	if err != nil {
+		log.Log.Fatalf("Could not rollback: %v", err)
+	}
+	log.Log.Info("Rolled back successfully.")
+}
+
+// Deletes a column from a table. All arguments are strings, to let them be standalone and not depending on any struct.
+func dropTableColum(x *xorm.Engine, tableName, col string) error {
+
+	switch viper.GetString("database.type") {
+	case "sqlite":
+		log.Log.Warning("Unable to drop columns in SQLite")
+	case "mysql":
+		_, err := x.Exec("ALTER TABLE " + tableName + " DROP COLUMN " + col)
+		if err != nil {
+			return err
+		}
+	default:
+		log.Log.Fatal("Unknown db.")
+	}
+	return nil
+}
+
+func initSchema(tx *xorm.Engine) error {
+	return tx.Sync2(
+		models.GetTables()...,
+	)
+}

pkg/models/fixtures/label_task.yml

@@ -1,3 +1,4 @@
 - id: 1
   task_id: 1
-  label_id: 4
+  label_id: 4
+  created: 0

pkg/models/fixtures/labels.yml

@@ -1,12 +1,20 @@
 - id: 1
   title: 'Label #1'
   created_by_id: 1
+  updated: 0
+  created: 0
 - id: 2
   title: 'Label #2'
   created_by_id: 1
+  updated: 0
+  created: 0
 - id: 3
   title: 'Label #3 - other user'
   created_by_id: 2
+  updated: 0
+  created: 0
 - id: 4
   title: 'Label #4 - visible via other task'
-  created_by_id: 2
+  created_by_id: 2
+  updated: 0
+  created: 0

pkg/models/fixtures/list.yml

@@ -4,27 +4,37 @@
   description: Lorem Ipsum
   owner_id: 1
   namespace_id: 1
+  updated: 0
+  created: 0
 -
   id: 2
   title: Test2
   description: Lorem Ipsum
   owner_id: 3
   namespace_id: 1
+  updated: 0
+  created: 0
 -
   id: 3
   title: Test3
   description: Lorem Ipsum
   owner_id: 3
   namespace_id: 2
+  updated: 0
+  created: 0
 -
   id: 4
   title: Test4
   description: Lorem Ipsum
   owner_id: 3
   namespace_id: 3
+  updated: 0
+  created: 0
 -
   id: 5
   title: Test5
   description: Lorem Ipsum
   owner_id: 5
-  namespace_id: 5
+  namespace_id: 5
+  updated: 0
+  created: 0

pkg/models/fixtures/namespaces.yml

@@ -3,13 +3,19 @@
   name: testnamespace
   description: Lorem Ipsum
   owner_id: 1
+  updated: 0
+  created: 0
 -
   id: 2
   name: testnamespace2
   description: Lorem Ipsum
   owner_id: 2
+  updated: 0
+  created: 0
 -
   id: 3
   name: testnamespace3
   description: Lorem Ipsum
-  owner_id: 3
+  owner_id: 3
+  updated: 0
+  created: 0

pkg/models/fixtures/team_list.yml

@@ -1,6 +1,10 @@
 - id: 1
   team_id: 1
   list_id: 3
+  updated: 0
+  created: 0
 - id: 2
   team_id: 2
   list_id: 3
+  updated: 0
+  created: 0

pkg/models/fixtures/team_members.yml

@@ -2,6 +2,8 @@
   team_id: 1
   user_id: 1
   admin: true
+  created: 0
 -
   team_id: 1
   user_id: 2
+  created: 0

pkg/models/fixtures/team_namespaces.yml

@@ -1,6 +1,10 @@
 - id: 1
   team_id: 1
   namespace_id: 3
+  updated: 0
+  created: 0
 - id: 2
   team_id: 2
   namespace_id: 3
+  updated: 0
+  created: 0

pkg/models/fixtures/users.yml

@@ -3,26 +3,36 @@
   username: 'user1'
   password: '1234'
   email: 'user1@example.com'
+  updated: 0
+  created: 0
 -
   id: 2
   username: 'user2'
   password: '1234'
   email: 'user2@example.com'
+  updated: 0
+  created: 0
 -
   id: 3
   username: 'user3'
   password: '1234'
   email: 'user3@example.com'
+  updated: 0
+  created: 0
 -
   id: 4
   username: 'user4'
   password: '1234'
   email: 'user4@example.com'
   email_confirm_token: tiepiQueed8ahc7zeeFe1eveiy4Ein8osooxegiephauph2Ael
+  updated: 0
+  created: 0
 -
   id: 5
   username: 'user5'
   password: '1234'
   email: 'user4@example.com'
   email_confirm_token: tiepiQueed8ahc7zeeFe1eveiy4Ein8osooxegiephauph2Ael
-  is_active: false
+  is_active: false
+  updated: 0
+  created: 0

pkg/models/fixtures/users_list.yml

@@ -1,6 +1,10 @@
 - id: 1
   user_id: 1
   list_id: 3
+  updated: 0
+  created: 0
 - id: 2
   user_id: 2
   list_id: 3
+  updated: 0
+  created: 0

pkg/models/fixtures/users_namespace.yml

@@ -1,6 +1,10 @@
 - id: 1
   user_id: 1
   namespace_id: 3
+  updated: 0
+  created: 0
 - id: 2
   user_id: 2
   namespace_id: 3
+  updated: 0
+  created: 0

pkg/models/label.go

@@ -27,18 +27,18 @@ type Label struct {
 	// The title of the lable. You'll see this one on tasks associated with it.
 	Title string `xorm:"varchar(250) not null" json:"title" valid:"runelength(3|250)" minLength:"3" maxLength:"250"`
 	// The label description.
-	Description string `xorm:"varchar(250)" json:"description" valid:"runelength(0|250)" maxLength:"250"`
+	Description string `xorm:"varchar(250) null" json:"description" valid:"runelength(0|250)" maxLength:"250"`
 	// The color this label has
-	HexColor string `xorm:"varchar(6)" json:"hex_color" valid:"runelength(0|6)" maxLength:"6"`
+	HexColor string `xorm:"varchar(6) null" json:"hex_color" valid:"runelength(0|6)" maxLength:"6"`
 
 	CreatedByID int64 `xorm:"int(11) not null" json:"-"`
 	// The user who created this label
 	CreatedBy *User `xorm:"-" json:"created_by"`
 
 	// A unix timestamp when this label was created. You cannot change this value.
-	Created int64 `xorm:"created" json:"created"`
+	Created int64 `xorm:"created not null" json:"created"`
 	// A unix timestamp when this label was last updated. You cannot change this value.
-	Updated int64 `xorm:"updated" json:"updated"`
+	Updated int64 `xorm:"updated not null" json:"updated"`
 
 	web.CRUDable `xorm:"-" json:"-"`
 	web.Rights   `xorm:"-" json:"-"`

pkg/models/label_task.go

@@ -29,7 +29,7 @@ type LabelTask struct {
 	// The label id you want to associate with a task.
 	LabelID int64 `xorm:"int(11) INDEX not null" json:"label_id" param:"label"`
 	// A unix timestamp when this task was created. You cannot change this value.
-	Created int64 `xorm:"created" json:"created"`
+	Created int64 `xorm:"created not null" json:"created"`
 
 	web.CRUDable `xorm:"-" json:"-"`
 	web.Rights   `xorm:"-" json:"-"`

pkg/models/list.go

@@ -25,11 +25,11 @@ type List struct {
 	// The unique, numeric id of this list.
 	ID int64 `xorm:"int(11) autoincr not null unique pk" json:"id" param:"list"`
 	// The title of the list. You'll see this in the namespace overview.
-	Title string `xorm:"varchar(250)" json:"title" valid:"required,runelength(3|250)" minLength:"3" maxLength:"250"`
+	Title string `xorm:"varchar(250) not null" json:"title" valid:"required,runelength(3|250)" minLength:"3" maxLength:"250"`
 	// The description of the list.
-	Description string `xorm:"varchar(1000)" json:"description" valid:"runelength(0|1000)" maxLength:"1000"`
-	OwnerID     int64  `xorm:"int(11) INDEX" json:"-"`
-	NamespaceID int64  `xorm:"int(11) INDEX" json:"-" param:"namespace"`
+	Description string `xorm:"varchar(1000) null" json:"description" valid:"runelength(0|1000)" maxLength:"1000"`
+	OwnerID     int64  `xorm:"int(11) INDEX not null" json:"-"`
+	NamespaceID int64  `xorm:"int(11) INDEX not null" json:"-" param:"namespace"`
 
 	// The user who created this list.
 	Owner User `xorm:"-" json:"owner" valid:"-"`
@@ -37,9 +37,9 @@ type List struct {
 	Tasks []*ListTask `xorm:"-" json:"tasks"`
 
 	// A unix timestamp when this list was created. You cannot change this value.
-	Created int64 `xorm:"created" json:"created"`
+	Created int64 `xorm:"created not null" json:"created"`
 	// A unix timestamp when this list was last updated. You cannot change this value.
-	Updated int64 `xorm:"updated" json:"updated"`
+	Updated int64 `xorm:"updated not null" json:"updated"`
 
 	web.CRUDable `xorm:"-" json:"-"`
 	web.Rights   `xorm:"-" json:"-"`

pkg/models/list_task_assignees.go

@@ -25,7 +25,7 @@ type ListTaskAssginee struct {
 	ID      int64 `xorm:"int(11) autoincr not null unique pk" json:"-"`
 	TaskID  int64 `xorm:"int(11) INDEX not null" json:"-" param:"listtask"`
 	UserID  int64 `xorm:"int(11) INDEX not null" json:"user_id" param:"user"`
-	Created int64 `xorm:"created"`
+	Created int64 `xorm:"created not null"`
 
 	web.CRUDable `xorm:"-" json:"-"`
 	web.Rights   `xorm:"-" json:"-"`

pkg/models/list_tasks.go

@@ -26,28 +26,28 @@ type ListTask struct {
 	// The unique, numeric id of this task.
 	ID int64 `xorm:"int(11) autoincr not null unique pk" json:"id" param:"listtask"`
 	// The task text. This is what you'll see in the list.
-	Text string `xorm:"varchar(250)" json:"text" valid:"runelength(3|250)" minLength:"3" maxLength:"250"`
+	Text string `xorm:"varchar(250) not null" json:"text" valid:"runelength(3|250)" minLength:"3" maxLength:"250"`
 	// The task description.
 	Description string `xorm:"varchar(250)" json:"description" valid:"runelength(0|250)" maxLength:"250"`
 	// Whether a task is done or not.
-	Done bool `xorm:"INDEX" json:"done"`
+	Done bool `xorm:"INDEX null" json:"done"`
 	// A unix timestamp when the task is due.
-	DueDateUnix int64 `xorm:"int(11) INDEX" json:"dueDate"`
+	DueDateUnix int64 `xorm:"int(11) INDEX null" json:"dueDate"`
 	// An array of unix timestamps when the user wants to be reminded of the task.
-	RemindersUnix []int64 `xorm:"JSON TEXT" json:"reminderDates"`
-	CreatedByID   int64   `xorm:"int(11)" json:"-"` // ID of the user who put that task on the list
+	RemindersUnix []int64 `xorm:"JSON TEXT null" json:"reminderDates"`
+	CreatedByID   int64   `xorm:"int(11) not null" json:"-"` // ID of the user who put that task on the list
 	// The list this task belongs to.
-	ListID int64 `xorm:"int(11) INDEX" json:"-" param:"list"`
+	ListID int64 `xorm:"int(11) INDEX not null" json:"-" param:"list"`
 	// An amount in seconds this task repeats itself. If this is set, when marking the task as done, it will mark itself as "undone" and then increase all remindes and the due date by its amount.
-	RepeatAfter int64 `xorm:"int(11) INDEX" json:"repeatAfter"`
+	RepeatAfter int64 `xorm:"int(11) INDEX null" json:"repeatAfter"`
 	// If the task is a subtask, this is the id of its parent.
-	ParentTaskID int64 `xorm:"int(11) INDEX" json:"parentTaskID"`
+	ParentTaskID int64 `xorm:"int(11) INDEX null" json:"parentTaskID"`
 	// The task priority. Can be anything you want, it is possible to sort by this later.
-	Priority int64 `xorm:"int(11)" json:"priority"`
+	Priority int64 `xorm:"int(11) null" json:"priority"`
 	// When this task starts.
-	StartDateUnix int64 `xorm:"int(11) INDEX" json:"startDate" query:"-"`
+	StartDateUnix int64 `xorm:"int(11) INDEX null" json:"startDate" query:"-"`
 	// When this task ends.
-	EndDateUnix int64 `xorm:"int(11) INDEX" json:"endDate" query:"-"`
+	EndDateUnix int64 `xorm:"int(11) INDEX null" json:"endDate" query:"-"`
 	// An array of users who are assigned to this task
 	Assignees []*User `xorm:"-" json:"assignees"`
 	// An array of labels which are associated with this task.
@@ -61,9 +61,9 @@ type ListTask struct {
 	Subtasks []*ListTask `xorm:"-" json:"subtasks"`
 
 	// A unix timestamp when this task was created. You cannot change this value.
-	Created int64 `xorm:"created" json:"created"`
+	Created int64 `xorm:"created not null" json:"created"`
 	// A unix timestamp when this task was last updated. You cannot change this value.
-	Updated int64 `xorm:"updated" json:"updated"`
+	Updated int64 `xorm:"updated not null" json:"updated"`
 
 	// The user who initially created the task.
 	CreatedBy User `xorm:"-" json:"createdBy" valid:"-"`

pkg/models/list_users.go

@@ -27,12 +27,12 @@ type ListUser struct {
 	// The list id.
 	ListID int64 `xorm:"int(11) not null INDEX" json:"-" param:"list"`
 	// The right this user has. 0 = Read only, 1 = Read & Write, 2 = Admin. See the docs for more details.
-	Right Right `xorm:"int(11) INDEX" json:"right" valid:"length(0|2)" maximum:"2" default:"0"`
+	Right Right `xorm:"int(11) INDEX null" json:"right" valid:"length(0|2)" maximum:"2" default:"0"`
 
 	// A unix timestamp when this relation was created. You cannot change this value.
-	Created int64 `xorm:"created" json:"created"`
+	Created int64 `xorm:"created not null" json:"created"`
 	// A unix timestamp when this relation was last updated. You cannot change this value.
-	Updated int64 `xorm:"updated" json:"updated"`
+	Updated int64 `xorm:"updated not null" json:"updated"`
 
 	web.CRUDable `xorm:"-" json:"-"`
 	web.Rights   `xorm:"-" json:"-"`

pkg/models/models.go

@@ -17,11 +17,10 @@
 package models
 
 import (
+	"code.vikunja.io/api/pkg/db"
 	"code.vikunja.io/api/pkg/log"
 	"encoding/gob"
-	"fmt"
 	_ "github.com/go-sql-driver/mysql" // Because.
-	"github.com/go-xorm/core"
 	"github.com/go-xorm/xorm"
 	xrc "github.com/go-xorm/xorm-redis-cache"
 	_ "github.com/mattn/go-sqlite3" // Because.
@@ -30,51 +29,11 @@ import (
 
 var (
 	x *xorm.Engine
-
-	tables            []interface{}
-	tablesWithPointer []interface{}
 )
 
-func getEngine() (*xorm.Engine, error) {
-	// Use Mysql if set
-	if viper.GetString("database.type") == "mysql" {
-		connStr := fmt.Sprintf(
-			"%s:%s@tcp(%s)/%s?charset=utf8&parseTime=true",
-			viper.GetString("database.user"),
-			viper.GetString("database.password"),
-			viper.GetString("database.host"),
-			viper.GetString("database.database"))
-		e, err := xorm.NewEngine("mysql", connStr)
-		e.SetMaxOpenConns(viper.GetInt("database.openconnections"))
-		return e, err
-	}
-
-	// Otherwise use sqlite
-	path := viper.GetString("database.path")
-	if path == "" {
-		path = "./db.db"
-	}
-	return xorm.NewEngine("sqlite3", path)
-}
-
-func init() {
-	tables = append(tables,
-		new(User),
-		new(List),
-		new(ListTask),
-		new(Team),
-		new(TeamMember),
-		new(TeamList),
-		new(TeamNamespace),
-		new(Namespace),
-		new(ListUser),
-		new(NamespaceUser),
-		new(ListTaskAssginee),
-		new(Label),
-		new(LabelTask),
-	)
-
-	tablesWithPointer = append(tables,
+// GetTables returns all structs which are also a table.
+func GetTables() []interface{} {
+	return []interface{}{
 		&User{},
 		&List{},
 		&ListTask{},
@@ -88,17 +47,19 @@ func init() {
 		&ListTaskAssginee{},
 		&Label{},
 		&LabelTask{},
-	)
+	}
 }
 
 // SetEngine sets the xorm.Engine
 func SetEngine() (err error) {
-	x, err = getEngine()
+	x, err = db.CreateDBEngine()
 	if err != nil {
-		return fmt.Errorf("failed to connect to database: %v", err)
+		log.Log.Criticalf("Could not connect to db: %v", err.Error())
+		return
 	}
 
 	// Cache
+	// We have to initialize the cache here to avoid import cycles
 	if viper.GetBool("cache.enabled") {
 		switch viper.GetString("cache.type") {
 		case "memory":
@@ -107,23 +68,12 @@ func SetEngine() (err error) {
 		case "redis":
 			cacher := xrc.NewRedisCacher(viper.GetString("redis.host"), viper.GetString("redis.password"), xrc.DEFAULT_EXPIRATION, x.Logger())
 			x.SetDefaultCacher(cacher)
-			gob.Register(tables)
-			gob.Register(tablesWithPointer) // Need to register tables with pointer as well...
+			gob.Register(GetTables())
 		default:
 			log.Log.Info("Did not find a valid cache type. Caching disabled. Please refer to the docs for poosible cache types.")
 		}
 	}
 
-	x.SetMapper(core.GonicMapper{})
-
-	// Sync dat shit
-	if err = x.StoreEngine("InnoDB").Sync2(tables...); err != nil {
-		return fmt.Errorf("sync database struct error: %v", err)
-	}
-
-	x.ShowSQL(viper.GetString("log.database") != "off")
-	x.SetLogger(xorm.NewSimpleLogger(log.GetLogWriter("database")))
-
 	return nil
 }
 

pkg/models/namespace.go

@@ -26,18 +26,18 @@ type Namespace struct {
 	// The unique, numeric id of this namespace.
 	ID int64 `xorm:"int(11) autoincr not null unique pk" json:"id" param:"namespace"`
 	// The name of this namespace.
-	Name string `xorm:"varchar(250)" json:"name" valid:"required,runelength(5|250)" minLength:"5" maxLength:"250"`
+	Name string `xorm:"varchar(250) not null" json:"name" valid:"required,runelength(5|250)" minLength:"5" maxLength:"250"`
 	// The description of the namespace
-	Description string `xorm:"varchar(1000)" json:"description" valid:"runelength(0|250)" maxLength:"250"`
+	Description string `xorm:"varchar(1000) null" json:"description" valid:"runelength(0|250)" maxLength:"250"`
 	OwnerID     int64  `xorm:"int(11) not null INDEX" json:"-"`
 
 	// The user who owns this namespace
 	Owner User `xorm:"-" json:"owner" valid:"-"`
 
 	// A unix timestamp when this namespace was created. You cannot change this value.
-	Created int64 `xorm:"created" json:"created"`
+	Created int64 `xorm:"created not null" json:"created"`
 	// A unix timestamp when this namespace was last updated. You cannot change this value.
-	Updated int64 `xorm:"updated" json:"updated"`
+	Updated int64 `xorm:"updated not null" json:"updated"`
 
 	web.CRUDable `xorm:"-" json:"-"`
 	web.Rights   `xorm:"-" json:"-"`
@@ -189,7 +189,7 @@ func (n *Namespace) ReadAll(search string, a web.Auth, page int) (interface{}, e
 
 	// Get all lists
 	lists := []*List{}
-	err = x.Table(&lists).
+	err = x.
 		In("namespace_id", namespaceids).
 		Find(&lists)
 	if err != nil {

pkg/models/namespace_users.go

@@ -27,12 +27,12 @@ type NamespaceUser struct {
 	// The namespace id
 	NamespaceID int64 `xorm:"int(11) not null INDEX" json:"-" param:"namespace"`
 	// The right this user has. 0 = Read only, 1 = Read & Write, 2 = Admin. See the docs for more details.
-	Right Right `xorm:"int(11) INDEX" json:"right" valid:"length(0|2)" maximum:"2" default:"0"`
+	Right Right `xorm:"int(11) INDEX null" json:"right" valid:"length(0|2)" maximum:"2" default:"0"`
 
 	// A unix timestamp when this relation was created. You cannot change this value.
-	Created int64 `xorm:"created" json:"created"`
+	Created int64 `xorm:"created not null" json:"created"`
 	// A unix timestamp when this relation was last updated. You cannot change this value.
-	Updated int64 `xorm:"updated" json:"updated"`
+	Updated int64 `xorm:"updated not null" json:"updated"`
 
 	web.CRUDable `xorm:"-" json:"-"`
 	web.Rights   `xorm:"-" json:"-"`

pkg/models/team_list.go

@@ -27,12 +27,12 @@ type TeamList struct {
 	// The list id.
 	ListID int64 `xorm:"int(11) not null INDEX" json:"-" param:"list"`
 	// The right this team has. 0 = Read only, 1 = Read & Write, 2 = Admin. See the docs for more details.
-	Right Right `xorm:"int(11) INDEX" json:"right" valid:"length(0|2)" maximum:"2" default:"0"`
+	Right Right `xorm:"int(11) INDEX null" json:"right" valid:"length(0|2)" maximum:"2" default:"0"`
 
 	// A unix timestamp when this relation was created. You cannot change this value.
-	Created int64 `xorm:"created" json:"created"`
+	Created int64 `xorm:"created not null" json:"created"`
 	// A unix timestamp when this relation was last updated. You cannot change this value.
-	Updated int64 `xorm:"updated" json:"updated"`
+	Updated int64 `xorm:"updated not null" json:"updated"`
 
 	web.CRUDable `xorm:"-" json:"-"`
 	web.Rights   `xorm:"-" json:"-"`

pkg/models/team_namespace.go

@@ -27,12 +27,12 @@ type TeamNamespace struct {
 	// The namespace id.
 	NamespaceID int64 `xorm:"int(11) not null INDEX" json:"-" param:"namespace"`
 	// The right this team has. 0 = Read only, 1 = Read & Write, 2 = Admin. See the docs for more details.
-	Right Right `xorm:"int(11) INDEX" json:"right" valid:"length(0|2)" maximum:"2" default:"0"`
+	Right Right `xorm:"int(11) INDEX null" json:"right" valid:"length(0|2)" maximum:"2" default:"0"`
 
 	// A unix timestamp when this relation was created. You cannot change this value.
-	Created int64 `xorm:"created" json:"created"`
+	Created int64 `xorm:"created not null" json:"created"`
 	// A unix timestamp when this relation was last updated. You cannot change this value.
-	Updated int64 `xorm:"updated" json:"updated"`
+	Updated int64 `xorm:"updated not null" json:"updated"`
 
 	web.CRUDable `xorm:"-" json:"-"`
 	web.Rights   `xorm:"-" json:"-"`

pkg/models/teams.go

@@ -69,10 +69,10 @@ type TeamMember struct {
 	// The id of the member.
 	UserID int64 `xorm:"int(11) not null INDEX" json:"userID" param:"user"`
 	// Whether or not the member is an admin of the team. See the docs for more about what a team admin can do
-	Admin bool `xorm:"tinyint(1) INDEX" json:"admin"`
+	Admin bool `xorm:"tinyint(1) INDEX null" json:"admin"`
 
 	// A unix timestamp when this relation was created. You cannot change this value.
-	Created int64 `xorm:"created" json:"created"`
+	Created int64 `xorm:"created not null" json:"created"`
 
 	web.CRUDable `xorm:"-" json:"-"`
 	web.Rights   `xorm:"-" json:"-"`

pkg/models/unit_tests.go

@@ -17,6 +17,7 @@
 package models
 
 import (
+	"code.vikunja.io/api/pkg/config"
 	_ "code.vikunja.io/api/pkg/config" // To trigger its init() which initializes the config
 	"code.vikunja.io/api/pkg/log"
 	"code.vikunja.io/api/pkg/mail"
@@ -51,12 +52,23 @@ func MainTest(m *testing.M, pathToRoot string) {
 
 func createTestEngine(fixturesDir string) error {
 	var err error
+	var fixturesHelper testfixtures.Helper = &testfixtures.SQLite{}
 	// If set, use the config we provided instead of normal
 	if os.Getenv("VIKUNJA_TESTS_USE_CONFIG") == "1" {
+		config.InitConfig()
 		err = SetEngine()
 		if err != nil {
 			return err
 		}
+
+		err = x.Sync2(GetTables()...)
+		if err != nil {
+			return err
+		}
+
+		if viper.GetString("database.type") == "mysql" {
+			fixturesHelper = &testfixtures.MySQL{}
+		}
 	} else {
 		x, err = xorm.NewEngine("sqlite3", "file::memory:?cache=shared")
 		if err != nil {
@@ -66,7 +78,7 @@ func createTestEngine(fixturesDir string) error {
 		x.SetMapper(core.GonicMapper{})
 
 		// Sync dat shit
-		if err := x.StoreEngine("InnoDB").Sync2(tables...); err != nil {
+		if err := x.Sync2(GetTables()...); err != nil {
 			return fmt.Errorf("sync database struct error: %v", err)
 		}
 
@@ -76,13 +88,6 @@ func createTestEngine(fixturesDir string) error {
 		}
 	}
 
-	var fixturesHelper testfixtures.Helper
-	if viper.GetString("database.type") == "mysql" {
-		fixturesHelper = &testfixtures.MySQL{}
-	} else {
-		fixturesHelper = &testfixtures.SQLite{}
-	}
-
 	return InitFixtures(fixturesHelper, fixturesDir)
 }
 

pkg/models/user.go

@@ -44,16 +44,16 @@ type User struct {
 	Username string `xorm:"varchar(250) not null unique" json:"username" valid:"length(3|250)" minLength:"3" maxLength:"250"`
 	Password string `xorm:"varchar(250) not null" json:"-"`
 	// The user's email address.
-	Email    string `xorm:"varchar(250)" json:"email" valid:"email,length(0|250)" maxLength:"250"`
-	IsActive bool   `json:"-"`
+	Email    string `xorm:"varchar(250) null" json:"email" valid:"email,length(0|250)" maxLength:"250"`
+	IsActive bool   `xorm:"null" json:"-"`
 
-	PasswordResetToken string `xorm:"varchar(450)" json:"-"`
-	EmailConfirmToken  string `xorm:"varchar(450)" json:"-"`
+	PasswordResetToken string `xorm:"varchar(450) null" json:"-"`
+	EmailConfirmToken  string `xorm:"varchar(450) null" json:"-"`
 
 	// A unix timestamp when this task was created. You cannot change this value.
-	Created int64 `xorm:"created" json:"created"`
+	Created int64 `xorm:"created not null" json:"created"`
 	// A unix timestamp when this task was last updated. You cannot change this value.
-	Updated int64 `xorm:"updated" json:"updated"`
+	Updated int64 `xorm:"updated not null" json:"updated"`
 
 	web.Auth `xorm:"-" json:"-"`
 }