ami/tl-vikunja
1
0
Fork 0
Code Activity

Add postgres support (#135)

Browse Source 
Revert fixture fixes for postgres

Use postgres connection string with spaces instead of url

Fix label order

Make postgres tests in ci less verbose

Add sequence update script

Skip resets in postgres

Remove option to skip resets in postgres

Make postgres tests in ci verboseq

Update test fixtures database

Fix file tests on postgres

Add postgres options to sample config

Make sure tests init test fixtures before running the actual tests

Fix issues with IDs too big to fit in an int

Fix duplicate auto incremented IDs

Refactor / Fix team tests

Refactor team member tests

Fix team member create

Fix label test

Fix getting labels

Fix test fixtures for postgresql

Fix connection string params

Disable ssl mode on postgres integration tests

Disable ssl mode on postgres tests

Use sprintf to create the connection string for postgresql

fixup! Add postgres support

Add postgres support

Added generate as a make dependency for make build

Clarify docs on building

Co-authored-by: kolaente <k@knt.li>
Co-authored-by: Jan Tojnar <jtojnar@gmail.com>
Reviewed-on: https://kolaente.dev/vikunja/api/pulls/135
This commit is contained in:
jtojnar
2020-02-16 21:42:04 +00:00
committed by konrad
parent 7e42724439
commit ce5be947b4
107 changed files with 9886 additions and 1595 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
pkg/db/db.go
View File

@ -21,7 +21,9 @@ import (
"code.vikunja.io/api/pkg/log"
"encoding/gob"
"fmt"
"net/url"
"strconv"
"strings"
"time"
"xorm.io/core"
"xorm.io/xorm"
@ -29,6 +31,7 @@ import (
xrc "gitea.com/xorm/xorm-redis-cache"
_ "github.com/go-sql-driver/mysql" // Because.
_ "github.com/lib/pq" // Because.
_ "github.com/mattn/go-sqlite3" // Because.
)
@ -53,6 +56,11 @@ func CreateDBEngine() (engine *xorm.Engine, err error) {
if err != nil {
return
}
} else if config.DatabaseType.GetString() == "postgres" {
engine, err = initPostgresEngine()
if err != nil {
return
}
} else {
// Otherwise use sqlite
engine, err = initSqliteEngine()
@ -113,6 +121,46 @@ func initMysqlEngine() (engine *xorm.Engine, err error) {
return
}
// parsePostgreSQLHostPort parses given input in various forms defined in
// https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING
// and returns proper host and port number.
func parsePostgreSQLHostPort(info string) (string, string) {
host, port := "127.0.0.1", "5432"
if strings.Contains(info, ":") && !strings.HasSuffix(info, "]") {
idx := strings.LastIndex(info, ":")
host = info[:idx]
port = info[idx+1:]
} else if len(info) > 0 {
host = info
}
return host, port
}
func initPostgresEngine() (engine *xorm.Engine, err error) {
host, port := parsePostgreSQLHostPort(config.DatabaseHost.GetString())
connStr := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=%s",
host,
port,
url.PathEscape(config.DatabaseUser.GetString()),
url.PathEscape(config.DatabasePassword.GetString()),
config.DatabaseDatabase.GetString(),
config.DatabaseSslMode.GetString(),
)
engine, err = xorm.NewEngine("postgres", connStr)
if err != nil {
return
}
engine.SetMaxOpenConns(config.DatabaseMaxOpenConnections.GetInt())
engine.SetMaxIdleConns(config.DatabaseMaxIdleConnections.GetInt())
max, err := time.ParseDuration(strconv.Itoa(config.DatabaseMaxConnectionLifetime.GetInt()) + `ms`)
if err != nil {
return
}
engine.SetConnMaxLifetime(max)
return
}
func initSqliteEngine() (engine *xorm.Engine, err error) {
path := config.DatabasePath.GetString()
if path == "" {

73
pkg/db/test_fixtures.go
View File

@ -19,41 +19,92 @@ package db
import (
"code.vikunja.io/api/pkg/config"
"fmt"
"github.com/go-testfixtures/testfixtures/v3"
"github.com/stretchr/testify/assert"
"gopkg.in/testfixtures.v2"
"path/filepath"
"testing"
)
var fixtures *testfixtures.Context
var fixtures *testfixtures.Loader
// InitFixtures initialize test fixtures for a test database
func InitFixtures(tablenames ...string) (err error) {
var helper testfixtures.Helper = &testfixtures.SQLite{}
if config.DatabaseType.GetString() == "mysql" {
helper = &testfixtures.MySQL{}
}
var testfiles func(loader *testfixtures.Loader) error
dir := filepath.Join(config.ServiceRootpath.GetString(), "pkg", "db", "fixtures")
testfixtures.SkipDatabaseNameCheck(true)
// If fixture table names are specified, load them
// Otherwise, load all fixtures
if len(tablenames) > 0 {
for i, name := range tablenames {
tablenames[i] = filepath.Join(dir, name+".yml")
}
fixtures, err = testfixtures.NewFiles(x.DB().DB, helper, tablenames...)
testfiles = testfixtures.Files(tablenames...)
} else {
fixtures, err = testfixtures.NewFolder(x.DB().DB, helper, dir)
testfiles = testfixtures.Directory(dir)
}
loaderOptions := []func(loader *testfixtures.Loader) error{
testfixtures.Database(x.DB().DB),
testfixtures.Dialect(config.DatabaseType.GetString()),
testfixtures.DangerousSkipTestDatabaseCheck(),
testfiles,
}
if config.DatabaseType.GetString() == "postgres" {
loaderOptions = append(loaderOptions, testfixtures.SkipResetSequences())
}
fixtures, err = testfixtures.New(loaderOptions...)
if err != nil {
return err
}
return err
}
// LoadFixtures load fixtures for a test database
func LoadFixtures() error {
return fixtures.Load()
err := fixtures.Load()
if err != nil {
return err
}
// Copied from https://github.com/go-gitea/gitea/blob/master/models/test_fixtures.go#L39
// Now if we're running postgres we need to tell it to update the sequences
if x.Dialect().DriverName() == "postgres" {
results, err := x.QueryString(`SELECT 'SELECT SETVAL(' ||
quote_literal(quote_ident(PGT.schemaname) || '.' || quote_ident(S.relname)) ||
', COALESCE(MAX(' ||quote_ident(C.attname)|| '), 1) ) FROM ' ||
quote_ident(PGT.schemaname)|| '.'||quote_ident(T.relname)|| ';'
FROM pg_class AS S,
pg_depend AS D,
pg_class AS T,
pg_attribute AS C,
pg_tables AS PGT
WHERE S.relkind = 'S'
AND S.oid = D.objid
AND D.refobjid = T.oid
AND D.refobjid = C.attrelid
AND D.refobjsubid = C.attnum
AND T.relname = PGT.tablename
ORDER BY S.relname;`)
if err != nil {
fmt.Printf("Failed to generate sequence update: %v\n", err)
return err
}
for _, r := range results {
for _, value := range r {
_, err = x.Exec(value)
if err != nil {
fmt.Printf("Failed to update sequence: %s Error: %v\n", value, err)
return err
}
}
}
}
return err
}
// LoadAndAssertFixtures loads all fixtures defined before and asserts they are correctly loaded