ami/tl-vikunja
1
0
Fork 0
Code Activity

Add prometheus endpoint for getting metrics (#33)

Browse Source
This commit is contained in:
konrad
2018-12-12 22:50:35 +00:00
committed by Gitea
parent ee398b5272
commit e047673c6b
189 changed files with 44128 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
pkg/config/config.go
View File

@ -17,6 +17,7 @@
package config
import (
"code.vikunja.io/api/pkg/log"
"crypto/rand"
"fmt"
"github.com/spf13/viper"
@ -26,13 +27,13 @@ import (
)
// InitConfig initializes the config, sets defaults etc.
func InitConfig() (err error) {
func init() {
// Set defaults
// Service config
random, err := random(32)
if err != nil {
return err
log.Log.Fatal(err.Error())
}
// Service
@ -46,6 +47,7 @@ func InitConfig() (err error) {
exPath := filepath.Dir(ex)
viper.SetDefault("service.rootpath", exPath)
viper.SetDefault("service.pagecount", 50)
viper.SetDefault("service.enablemetrics", false)
// Database
viper.SetDefault("database.type", "sqlite")
viper.SetDefault("database.host", "localhost")
@ -59,8 +61,6 @@ func InitConfig() (err error) {
viper.SetDefault("cache.enabled", false)
viper.SetDefault("cache.type", "memory")
viper.SetDefault("cache.maxelementsize", 1000)
viper.SetDefault("cache.redishost", "localhost:6379")
viper.SetDefault("cache.redispassword", "")
// Mailer
viper.SetDefault("mailer.host", "")
viper.SetDefault("mailer.port", "587")
@ -70,6 +70,11 @@ func InitConfig() (err error) {
viper.SetDefault("mailer.fromemail", "mail@vikunja")
viper.SetDefault("mailer.queuelength", 100)
viper.SetDefault("mailer.queuetimeout", 30)
// Redis
viper.SetDefault("redis.enabled", false)
viper.SetDefault("redis.host", "localhost:6379")
viper.SetDefault("redis.password", "")
viper.SetDefault("redis.db", 0)
// Init checking for environment variables
viper.SetEnvPrefix("vikunja")
@ -81,11 +86,9 @@ func InitConfig() (err error) {
viper.SetConfigName("config")
err = viper.ReadInConfig()
if err != nil {
fmt.Println(err)
fmt.Println("Using defaults.")
log.Log.Info(err)
log.Log.Info("Using defaults.")
}
return nil
}
func random(length int) (string, error) {

92
pkg/metrics/active_users.go Normal file
View File

@ -0,0 +1,92 @@
// Vikunja is a todo-list application to facilitate your life.
// Copyright 2018 Vikunja and contributors. All rights reserved.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package metrics
import (
"bytes"
"code.vikunja.io/api/pkg/log"
"encoding/gob"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"time"
)
// SecondsUntilInactive defines the seconds until a user is considered inactive
const SecondsUntilInactive = 60
// ActiveUsersKey is the key used to store active users in redis
const ActiveUsersKey = `activeusers`
// ActiveUser defines an active user
type ActiveUser struct {
UserID int64
LastSeen time.Time
}
func init() {
promauto.NewGaugeFunc(prometheus.GaugeOpts{
Name: "vikunja_active_users",
Help: "The currently active users on this node",
}, func() float64 {
allActiveUsers, err := GetActiveUsers()
if err != nil {
log.Log.Error(err.Error())
}
activeUsersCount := 0
for _, u := range allActiveUsers {
if time.Since(u.LastSeen) < SecondsUntilInactive*time.Second {
activeUsersCount++
}
}
return float64(activeUsersCount)
})
}
// GetActiveUsers returns the active users from redis
func GetActiveUsers() (users []*ActiveUser, err error) {
activeUsersR, err := r.Get(ActiveUsersKey).Bytes()
if err != nil {
if err.Error() == "redis: nil" {
return users, nil
}
return
}
var b bytes.Buffer
_, err = b.Write(activeUsersR)
if err != nil {
return nil, err
}
d := gob.NewDecoder(&b)
if err := d.Decode(&users); err != nil {
return nil, err
}
return
}
// SetActiveUsers sets the active users from redis
func SetActiveUsers(users []*ActiveUser) (err error) {
var b bytes.Buffer
e := gob.NewEncoder(&b)
if err := e.Encode(users); err != nil {
return err
}
return r.Set(ActiveUsersKey, b.Bytes(), 0).Err()
}

123
pkg/metrics/metrics.go Normal file
View File

@ -0,0 +1,123 @@
// Vikunja is a todo-list application to facilitate your life.
// Copyright 2018 Vikunja and contributors. All rights reserved.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package metrics
import (
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/red"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/spf13/viper"
)
var r = red.GetRedis()
const (
// ListCountKey is the name of the key in which we save the list count
ListCountKey = `listcount`
// UserCountKey is the name of the key we use to store total users in redis
UserCountKey = `usercount`
// NamespaceCountKey is the name of the key we use to store the amount of total namespaces in redis
NamespaceCountKey = `namespacecount`
// TaskCountKey is the name of the key we use to store the amount of total tasks in redis
TaskCountKey = `taskcount`
// TeamCountKey is the name of the key we use to store the amount of total teams in redis
TeamCountKey = `teamcount`
)
func init() {
// Register total list count metric
promauto.NewGaugeFunc(prometheus.GaugeOpts{
Name: "vikunja_list_count",
Help: "The number of lists on this instance",
}, func() float64 {
count, _ := GetCount(ListCountKey)
return float64(count)
})
// Register total user count metric
promauto.NewGaugeFunc(prometheus.GaugeOpts{
Name: "vikunja_user_count",
Help: "The total number of users on this instance",
}, func() float64 {
count, _ := GetCount(UserCountKey)
return float64(count)
})
// Register total Namespaces count metric
promauto.NewGaugeFunc(prometheus.GaugeOpts{
Name: "vikunja_namespcae_count",
Help: "The total number of namespaces on this instance",
}, func() float64 {
count, _ := GetCount(NamespaceCountKey)
return float64(count)
})
// Register total Tasks count metric
promauto.NewGaugeFunc(prometheus.GaugeOpts{
Name: "vikunja_task_count",
Help: "The total number of tasks on this instance",
}, func() float64 {
count, _ := GetCount(TaskCountKey)
return float64(count)
})
// Register total user count metric
promauto.NewGaugeFunc(prometheus.GaugeOpts{
Name: "vikunja_team_count",
Help: "The total number of teams on this instance",
}, func() float64 {
count, _ := GetCount(TeamCountKey)
return float64(count)
})
}
// GetCount returns the current count from redis
func GetCount(key string) (count int64, err error) {
count, err = r.Get(key).Int64()
if err != nil && err.Error() != "redis: nil" {
return
}
err = nil
return
}
// SetCount sets the list count to a given value
func SetCount(count int64, key string) error {
return r.Set(key, count, 0).Err()
}
// UpdateCount updates a count with a given amount
func UpdateCount(update int64, key string) {
if !viper.GetBool("service.enablemetrics") {
return
}
oldtotal, err := GetCount(key)
if err != nil {
log.Log.Error(err.Error())
}
err = SetCount(oldtotal+update, key)
if err != nil {
log.Log.Error(err.Error())
}
}

6
pkg/models/list_create_update.go
View File

@ -16,7 +16,10 @@
package models
import "code.vikunja.io/web"
import (
"code.vikunja.io/api/pkg/metrics"
"code.vikunja.io/web"
)
// CreateOrUpdateList updates a list or creates it if it doesn't exist
func CreateOrUpdateList(list *List) (err error) {
@ -36,6 +39,7 @@ func CreateOrUpdateList(list *List) (err error) {
if list.ID == 0 {
_, err = x.Insert(list)
metrics.UpdateCount(1, metrics.ListCountKey)
} else {
_, err = x.ID(list.ID).Update(list)
}

6
pkg/models/list_delete.go
View File

@ -16,7 +16,10 @@
package models
import _ "code.vikunja.io/web" // For swaggerdocs generation
import (
"code.vikunja.io/api/pkg/metrics"
_ "code.vikunja.io/web" // For swaggerdocs generation
)
// Delete implements the delete method of CRUDable
// @Summary Deletes a list
@ -41,6 +44,7 @@ func (l *List) Delete() (err error) {
if err != nil {
return
}
metrics.UpdateCount(-1, metrics.ListCountKey)
// Delete all todotasks on that list
_, err = x.Where("list_id = ?", l.ID).Delete(&ListTask{})

9
pkg/models/list_tasks_create_update.go
View File

@ -17,6 +17,7 @@
package models
import (
"code.vikunja.io/api/pkg/metrics"
"code.vikunja.io/web"
"github.com/imdario/mergo"
)
@ -61,8 +62,12 @@ func (i *ListTask) Create(a web.Auth) (err error) {
i.CreatedByID = u.ID
i.CreatedBy = u
_, err = x.Insert(i)
return err
if _, err = x.Insert(i); err != nil {
return err
}
metrics.UpdateCount(1, metrics.TaskCountKey)
return
}
// Update updates a list task

11
pkg/models/list_tasks_delete.go
View File

@ -16,7 +16,10 @@
package models
import _ "code.vikunja.io/web" // For swaggerdocs generation
import (
"code.vikunja.io/api/pkg/metrics"
_ "code.vikunja.io/web" // For swaggerdocs generation
)
// Delete implements the delete method for listTask
// @Summary Delete a task
@ -38,6 +41,10 @@ func (i *ListTask) Delete() (err error) {
return
}
_, err = x.ID(i.ID).Delete(ListTask{})
if _, err = x.ID(i.ID).Delete(ListTask{}); err != nil {
return err
}
metrics.UpdateCount(-1, metrics.TaskCountKey)
return
}

10
pkg/models/models.go
View File

@ -17,14 +17,13 @@
package models
import (
"encoding/gob"
"fmt"
_ "github.com/go-sql-driver/mysql" // Because.
"github.com/go-xorm/core"
"github.com/go-xorm/xorm"
xrc "github.com/go-xorm/xorm-redis-cache"
_ "github.com/mattn/go-sqlite3" // Because.
"encoding/gob"
"github.com/spf13/viper"
)
@ -86,7 +85,7 @@ func SetEngine() (err error) {
x.SetDefaultCacher(cacher)
break
case "redis":
cacher := xrc.NewRedisCacher(viper.GetString("cache.redishost"), viper.GetString("cache.redispassword"), xrc.DEFAULT_EXPIRATION, x.Logger())
cacher := xrc.NewRedisCacher(viper.GetString("redis.host"), viper.GetString("redis.password"), xrc.DEFAULT_EXPIRATION, x.Logger())
x.SetDefaultCacher(cacher)
gob.Register(tables)
break
@ -118,3 +117,8 @@ func getLimitFromPageIndex(page int) (limit, start int) {
start = limit * (page - 1)
return
}
// GetTotalCount returns the total amount of something
func GetTotalCount(counting interface{}) (count int64, err error) {
return x.Count(counting)
}

11
pkg/models/namespace_create.go
View File

@ -16,7 +16,10 @@
package models
import "code.vikunja.io/web"
import (
"code.vikunja.io/api/pkg/metrics"
"code.vikunja.io/web"
)
// Create implements the creation method via the interface
// @Summary Creates a new namespace
@ -51,6 +54,10 @@ func (n *Namespace) Create(a web.Auth) (err error) {
n.OwnerID = n.Owner.ID
// Insert
_, err = x.Insert(n)
if _, err = x.Insert(n); err != nil {
return err
}
metrics.UpdateCount(1, metrics.NamespaceCountKey)
return
}

7
pkg/models/namespace_delete.go
View File

@ -16,7 +16,10 @@
package models
import _ "code.vikunja.io/web" // For swaggerdocs generation
import (
"code.vikunja.io/api/pkg/metrics"
_ "code.vikunja.io/web" // For swaggerdocs generation
)
// Delete deletes a namespace
// @Summary Deletes a namespace
@ -66,5 +69,7 @@ func (n *Namespace) Delete() (err error) {
return
}
metrics.UpdateCount(-1, metrics.NamespaceCountKey)
return
}

11
pkg/models/teams_create.go
View File

@ -16,7 +16,10 @@
package models
import "code.vikunja.io/web"
import (
"code.vikunja.io/api/pkg/metrics"
"code.vikunja.io/web"
)
// Create is the handler to create a team
// @Summary Creates a new team
@ -51,6 +54,10 @@ func (t *Team) Create(a web.Auth) (err error) {
// Insert the current user as member and admin
tm := TeamMember{TeamID: t.ID, UserID: doer.ID, Admin: true}
err = tm.Create(doer)
if err = tm.Create(doer); err != nil {
return err
}
metrics.UpdateCount(1, metrics.TeamCountKey)
return
}

10
pkg/models/teams_delete.go
View File

@ -16,7 +16,10 @@
package models
import _ "code.vikunja.io/web" // For swaggerdocs generation
import (
"code.vikunja.io/api/pkg/metrics"
_ "code.vikunja.io/web" // For swaggerdocs generation
)
// Delete deletes a team
// @Summary Deletes a team
@ -57,5 +60,10 @@ func (t *Team) Delete() (err error) {
// Delete team <-> lists relations
_, err = x.Where("team_id = ?", t.ID).Delete(&TeamList{})
if err != nil {
return
}
metrics.UpdateCount(-1, metrics.TeamCountKey)
return
}

9
pkg/models/unit_tests.go
View File

@ -17,6 +17,8 @@
package models
import (
_ "code.vikunja.io/api/pkg/config" // To trigger its init() which initializes the config
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/mail"
"fmt"
"github.com/go-xorm/core"
@ -36,8 +38,7 @@ func MainTest(m *testing.M, pathToRoot string) {
var err error
fixturesDir := filepath.Join(pathToRoot, "models", "fixtures")
if err = createTestEngine(fixturesDir); err != nil {
fmt.Fprintf(os.Stderr, "Error creating test engine: %v\n", err)
os.Exit(1)
log.Log.Fatalf("Error creating test engine: %v\n", err)
}
IsTesting = true
@ -46,7 +47,9 @@ func MainTest(m *testing.M, pathToRoot string) {
mail.StartMailDaemon()
// Create test database
PrepareTestDatabase()
if err = PrepareTestDatabase(); err != nil {
log.Log.Fatal(err.Error())
}
os.Exit(m.Run())
}

29
pkg/models/user.go
View File

@ -18,12 +18,14 @@ package models
import (
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/metrics"
"code.vikunja.io/web"
"fmt"
"github.com/dgrijalva/jwt-go"
"github.com/labstack/echo"
"golang.org/x/crypto/bcrypt"
"reflect"
"time"
)
// UserLogin Object to recive user credentials in JSON format
@ -159,3 +161,30 @@ func GetCurrentUser(c echo.Context) (user *User, err error) {
return
}
// UpdateActiveUsersFromContext updates the currently active users in redis
func UpdateActiveUsersFromContext(c echo.Context) (err error) {
user, err := GetCurrentUser(c)
if err != nil {
return err
}
allActiveUsers, err := metrics.GetActiveUsers()
if err != nil {
return
}
var uupdated bool
for in, u := range allActiveUsers {
if u.UserID == user.ID {
allActiveUsers[in].LastSeen = time.Now()
uupdated = true
}
}
if !uupdated {
allActiveUsers = append(allActiveUsers, &metrics.ActiveUser{UserID: user.ID, LastSeen: time.Now()})
}
return metrics.SetActiveUsers(allActiveUsers)
}

4
pkg/models/user_add_update.go
View File

@ -18,6 +18,7 @@ package models
import (
"code.vikunja.io/api/pkg/mail"
"code.vikunja.io/api/pkg/metrics"
"code.vikunja.io/api/pkg/utils"
"golang.org/x/crypto/bcrypt"
)
@ -78,6 +79,9 @@ func CreateUser(user User) (newUser User, err error) {
return User{}, err
}
// Update the metrics
metrics.UpdateCount(1, metrics.ActiveUsersKey)
// Get the full new User
newUserOut, err := GetUser(newUser)
if err != nil {

5
pkg/models/user_delete.go
View File

@ -16,6 +16,8 @@
package models
import "code.vikunja.io/api/pkg/metrics"
// DeleteUserByID deletes a user by its ID
func DeleteUserByID(id int64, doer *User) error {
// Check if the id is 0
@ -30,5 +32,8 @@ func DeleteUserByID(id int64, doer *User) error {
return err
}
// Update the metrics
metrics.UpdateCount(-1, metrics.ActiveUsersKey)
return err
}

52
pkg/red/redis.go Normal file
View File

@ -0,0 +1,52 @@
// Vikunja is a todo-list application to facilitate your life.
// Copyright 2018 Vikunja and contributors. All rights reserved.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package red
import (
"code.vikunja.io/api/pkg/log"
"github.com/go-redis/redis"
"github.com/spf13/viper"
)
var r *redis.Client
// SetRedis initializes a redis connection
func init() {
if !viper.GetBool("redis.enabled") {
return
}
if viper.GetString("redis.host") == "" {
log.Log.Fatal("No redis host provided.")
}
r = redis.NewClient(&redis.Options{
Addr: viper.GetString("redis.host"),
Password: viper.GetString("redis.password"),
DB: viper.GetInt("redis.db"),
})
err := r.Ping().Err()
if err != nil {
log.Log.Fatal(err.Error())
}
}
// GetRedis returns a pointer to a redis client
func GetRedis() *redis.Client {
return r
}

65
pkg/routes/routes.go
View File

@ -45,6 +45,7 @@ package routes
import (
_ "code.vikunja.io/api/docs" // To generate swagger docs
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/metrics"
"code.vikunja.io/api/pkg/models"
apiv1 "code.vikunja.io/api/pkg/routes/api/v1"
"code.vikunja.io/web"
@ -52,6 +53,7 @@ import (
"github.com/asaskevich/govalidator"
"github.com/labstack/echo"
"github.com/labstack/echo/middleware"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/spf13/viper"
"github.com/swaggo/echo-swagger"
)
@ -112,6 +114,54 @@ func RegisterRoutes(e *echo.Echo) {
// Swagger UI
a.GET("/swagger/*", echoSwagger.WrapHandler)
// Prometheus endpoint
if viper.GetBool("service.enablemetrics") {
if !viper.GetBool("redis.enabled") {
log.Log.Fatal("You have to enable redis in order to use metrics")
}
type countable struct {
Rediskey string
Type interface{}
}
for _, c := range []countable{
{
metrics.ListCountKey,
models.List{},
},
{
metrics.UserCountKey,
models.User{},
},
{
metrics.NamespaceCountKey,
models.Namespace{},
},
{
metrics.TaskCountKey,
models.ListTask{},
},
{
metrics.TeamCountKey,
models.Team{},
},
} {
// Set initial totals
total, err := models.GetTotalCount(c.Type)
if err != nil {
log.Log.Fatalf("Could not set initial count for %v, error was %s", c.Type, err)
}
if err := metrics.SetCount(total, c.Rediskey); err != nil {
log.Log.Fatalf("Could not set initial count for %v, error was %s", c.Type, err)
}
}
a.GET("/metrics", echo.WrapHandler(promhttp.Handler()))
}
// User stuff
a.POST("/login", apiv1.Login)
a.POST("/register", apiv1.RegisterUser)
a.POST("/user/password/token", apiv1.UserRequestResetPasswordToken)
@ -138,6 +188,21 @@ func RegisterRoutes(e *echo.Echo) {
}
})
// Middleware to collect metrics
if viper.GetBool("service.enablemetrics") {
a.Use(func(next echo.HandlerFunc) echo.HandlerFunc {
return func(c echo.Context) error {
// Update currently active users
if err := models.UpdateActiveUsersFromContext(c); err != nil {
log.Log.Error(err)
return next(c)
}
return next(c)
}
})
}
a.POST("/tokenTest", apiv1.CheckToken)
// User stuff