ami/tl-vikunja
1
0
Fork 0
Code

feat(tasks): add searching via typesense

Browse Source
This commit is contained in:
kolaente 2023-08-28 12:14:50 +02:00
parent dee46d527a
commit 010b4ce783
No known key found for this signature in database
GPG Key ID: F40E70337AB24C9B
8 changed files with 308 additions and 201 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
pkg/cmd/index.go
View File

@ -40,6 +40,8 @@ var indexCmd = &cobra.Command{
return
}
log.Infof("Indexing… This may take a while.")
err := models.CreateTypesenseCollections()
if err != nil {
log.Critical(err.Error())
@ -48,6 +50,9 @@ var indexCmd = &cobra.Command{
err = models.ReindexAllTasks()
if err != nil {
log.Critical(err.Error())
return
}
log.Infof("Done!")
},
}

2
pkg/models/export.go
View File

@ -153,7 +153,7 @@ func exportProjectsAndTasks(s *xorm.Session, u *user.User, wr *zip.Writer) (task
projectIDs = append(projectIDs, p.ID)
}
tasks, _, _, err := getTasksForProjects(s, rawProjects, u, &taskOptions{
tasks, _, _, err := getTasksForProjects(s, rawProjects, u, &taskSearchOptions{
page: 0,
perPage: -1,
})

2
pkg/models/project.go
View File

@ -969,7 +969,7 @@ func (p *Project) Delete(s *xorm.Session, a web.Auth) (err error) {
// Delete all tasks on that project
// Using the loop to make sure all related entities to all tasks are properly deleted as well.
tasks, _, _, err := getRawTasksForProjects(s, []*Project{p}, a, &taskOptions{})
tasks, _, _, err := getRawTasksForProjects(s, []*Project{p}, a, &taskSearchOptions{})
if err != nil {
return
}

2
pkg/models/project_duplicate.go
View File

@ -209,7 +209,7 @@ func (pd *ProjectDuplicate) Create(s *xorm.Session, doer web.Auth) (err error) {
func duplicateTasks(s *xorm.Session, doer web.Auth, ld *ProjectDuplicate, bucketMap map[int64]int64) (err error) {
// Get all tasks + all task details
tasks, _, _, err := getTasksForProjects(s, []*Project{{ID: ld.ProjectID}}, doer, &taskOptions{})
tasks, _, _, err := getTasksForProjects(s, []*Project{{ID: ld.ProjectID}}, doer, &taskSearchOptions{})
if err != nil {
return err
}

4
pkg/models/task_collection.go
View File

@ -80,7 +80,7 @@ func validateTaskField(fieldName string) error {
return ErrInvalidTaskField{TaskField: fieldName}
}
func getTaskFilterOptsFromCollection(tf *TaskCollection) (opts *taskOptions, err error) {
func getTaskFilterOptsFromCollection(tf *TaskCollection) (opts *taskSearchOptions, err error) {
if len(tf.SortByArr) > 0 {
tf.SortBy = append(tf.SortBy, tf.SortByArr...)
}
@ -108,7 +108,7 @@ func getTaskFilterOptsFromCollection(tf *TaskCollection) (opts *taskOptions, err
sort = append(sort, param)
}
opts = &taskOptions{
opts = &taskSearchOptions{
sortby: sort,
filterConcat: taskFilterConcatinator(tf.FilterConcat),
filterIncludeNulls: tf.FilterIncludeNulls,

280
pkg/models/task_search.go Normal file
View File

@ -0,0 +1,280 @@
// Vikunja is a to-do list application to facilitate your life.
// Copyright 2018-2023 Vikunja and contributors. All rights reserved.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public Licensee as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public Licensee for more details.
//
// You should have received a copy of the GNU Affero General Public Licensee
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package models
import (
"code.vikunja.io/api/pkg/db"
"code.vikunja.io/web"
"github.com/typesense/typesense-go/typesense/api"
"github.com/typesense/typesense-go/typesense/api/pointer"
"strconv"
"strings"
"xorm.io/builder"
"xorm.io/xorm"
"xorm.io/xorm/schemas"
)
type taskSearcher interface {
Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error)
}
type dbTaskSearcher struct {
s *xorm.Session
a web.Auth
hasFavoritesProject bool
}
func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) {
// Since xorm does not use placeholders for order by, it is possible to expose this with sql injection if we're directly
// passing user input to the db.
// As a workaround to prevent this, we check for valid column names here prior to passing it to the db.
var orderby string
for i, param := range opts.sortby {
// Validate the params
if err := param.validate(); err != nil {
return nil, totalCount, err
}
// Mysql sorts columns with null values before ones without null value.
// Because it does not have support for NULLS FIRST or NULLS LAST we work around this by
// first sorting for null (or not null) values and then the order we actually want to.
if db.Type() == schemas.MYSQL {
orderby += "`" + param.sortBy + "` IS NULL, "
}
orderby += "`" + param.sortBy + "` " + param.orderBy.String()
// Postgres and sqlite allow us to control how columns with null values are sorted.
// To make that consistent with the sort order we have and other dbms, we're adding a separate clause here.
if db.Type() == schemas.POSTGRES || db.Type() == schemas.SQLITE {
orderby += " NULLS LAST"
}
if (i + 1) < len(opts.sortby) {
orderby += ", "
}
}
// Some filters need a special treatment since they are in a separate table
reminderFilters := []builder.Cond{}
assigneeFilters := []builder.Cond{}
labelFilters := []builder.Cond{}
projectFilters := []builder.Cond{}
var filters = make([]builder.Cond, 0, len(opts.filters))
// To still find tasks with nil values, we exclude 0s when comparing with >/< values.
for _, f := range opts.filters {
if f.field == "reminders" {
f.field = "reminder" // This is the name in the db
filter, err := getFilterCond(f, opts.filterIncludeNulls)
if err != nil {
return nil, totalCount, err
}
reminderFilters = append(reminderFilters, filter)
continue
}
if f.field == "assignees" {
if f.comparator == taskFilterComparatorLike {
return nil, totalCount, err
}
f.field = "username"
filter, err := getFilterCond(f, opts.filterIncludeNulls)
if err != nil {
return nil, totalCount, err
}
assigneeFilters = append(assigneeFilters, filter)
continue
}
if f.field == "labels" || f.field == "label_id" {
f.field = "label_id"
filter, err := getFilterCond(f, opts.filterIncludeNulls)
if err != nil {
return nil, totalCount, err
}
labelFilters = append(labelFilters, filter)
continue
}
if f.field == "parent_project" || f.field == "parent_project_id" {
f.field = "parent_project_id"
filter, err := getFilterCond(f, opts.filterIncludeNulls)
if err != nil {
return nil, totalCount, err
}
projectFilters = append(projectFilters, filter)
continue
}
filter, err := getFilterCond(f, opts.filterIncludeNulls)
if err != nil {
return nil, totalCount, err
}
filters = append(filters, filter)
}
// Then return all tasks for that projects
var where builder.Cond
if opts.search != "" {
where =
builder.Or(
db.ILIKE("title", opts.search),
db.ILIKE("description", opts.search),
)
searchIndex := getTaskIndexFromSearchString(opts.search)
if searchIndex > 0 {
where = builder.Or(where, builder.Eq{"`index`": searchIndex})
}
}
var projectIDCond builder.Cond
var favoritesCond builder.Cond
if len(opts.projectIDs) > 0 {
projectIDCond = builder.In("project_id", opts.projectIDs)
}
if d.hasFavoritesProject {
// All favorite tasks for that user
favCond := builder.
Select("entity_id").
From("favorites").
Where(
builder.And(
builder.Eq{"user_id": d.a.GetID()},
builder.Eq{"kind": FavoriteKindTask},
))
favoritesCond = builder.In("id", favCond)
}
if len(reminderFilters) > 0 {
filters = append(filters, getFilterCondForSeparateTable("task_reminders", opts.filterConcat, reminderFilters))
}
if len(assigneeFilters) > 0 {
assigneeFilter := []builder.Cond{
builder.In("user_id",
builder.Select("id").
From("users").
Where(builder.Or(assigneeFilters...)),
)}
filters = append(filters, getFilterCondForSeparateTable("task_assignees", opts.filterConcat, assigneeFilter))
}
if len(labelFilters) > 0 {
filters = append(filters, getFilterCondForSeparateTable("label_tasks", opts.filterConcat, labelFilters))
}
if len(projectFilters) > 0 {
var filtercond builder.Cond
if opts.filterConcat == filterConcatOr {
filtercond = builder.Or(projectFilters...)
}
if opts.filterConcat == filterConcatAnd {
filtercond = builder.And(projectFilters...)
}
cond := builder.In(
"project_id",
builder.
Select("id").
From("projects").
Where(filtercond),
)
filters = append(filters, cond)
}
var filterCond builder.Cond
if len(filters) > 0 {
if opts.filterConcat == filterConcatOr {
filterCond = builder.Or(filters...)
}
if opts.filterConcat == filterConcatAnd {
filterCond = builder.And(filters...)
}
}
limit, start := getLimitFromPageIndex(opts.page, opts.perPage)
cond := builder.And(builder.Or(projectIDCond, favoritesCond), where, filterCond)
query := d.s.Where(cond)
if limit > 0 {
query = query.Limit(limit, start)
}
tasks = []*Task{}
err = query.OrderBy(orderby).Find(&tasks)
if err != nil {
return nil, totalCount, err
}
queryCount := d.s.Where(cond)
totalCount, err = queryCount.
Count(&Task{})
if err != nil {
return nil, totalCount, err
}
return
}
type typesenseTaskSearcher struct {
s *xorm.Session
}
func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) {
projectIDStrings := []string{}
for _, id := range opts.projectIDs {
projectIDStrings = append(projectIDStrings, strconv.FormatInt(id, 10))
}
params := &api.SearchCollectionParams{
Q: opts.search,
QueryBy: "title, description, comments.comment",
Page: pointer.Int(opts.page),
PerPage: pointer.Int(opts.perPage),
ExhaustiveSearch: pointer.True(),
FilterBy: pointer.String("project_id: [" + strings.Join(projectIDStrings, ", ") + "]"),
}
result, err := typesenseClient.Collection("tasks").
Documents().
Search(params)
if err != nil {
return
}
taskIDs := []int64{}
for _, h := range *result.Hits {
hit := *h.Document
taskID, err := strconv.ParseInt(hit["id"].(string), 10, 64)
if err != nil {
return nil, 0, err
}
taskIDs = append(taskIDs, taskID)
}
tasks = []*Task{}
err = t.s.In("id", taskIDs).Find(&tasks)
return tasks, int64(*result.Found), err
}

210
pkg/models/tasks.go
View File

@ -25,7 +25,6 @@ import (
"time"
"code.vikunja.io/api/pkg/config"
"code.vikunja.io/api/pkg/db"
"code.vikunja.io/api/pkg/events"
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/user"
@ -36,7 +35,6 @@ import (
"github.com/jinzhu/copier"
"xorm.io/builder"
"xorm.io/xorm"
"xorm.io/xorm/schemas"
)
type TaskRepeatMode int
@ -167,7 +165,7 @@ const (
filterConcatOr = "or"
)
type taskOptions struct {
type taskSearchOptions struct {
search string
page int
perPage int
@ -175,6 +173,7 @@ type taskOptions struct {
filters []*taskFilter
filterConcat taskFilterConcatinator
filterIncludeNulls bool
projectIDs []int64
}
// ReadAll is a dummy function to still have that endpoint documented
@ -266,7 +265,7 @@ func getTaskIndexFromSearchString(s string) (index int64) {
}
//nolint:gocyclo
func getRawTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, opts *taskOptions) (tasks []*Task, resultCount int, totalItems int64, err error) {
func getRawTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, opts *taskSearchOptions) (tasks []*Task, resultCount int, totalItems int64, err error) {
// If the user does not have any projects, don't try to get any tasks
if len(projects) == 0 {
@ -279,14 +278,14 @@ func getRawTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, op
}
// Get all project IDs and get the tasks
var projectIDs []int64
opts.projectIDs = []int64{}
var hasFavoritesProject bool
for _, l := range projects {
if l.ID == FavoritesPseudoProject.ID {
for _, p := range projects {
if p.ID == FavoritesPseudoProject.ID {
hasFavoritesProject = true
continue
}
projectIDs = append(projectIDs, l.ID)
opts.projectIDs = append(opts.projectIDs, p.ID)
}
// Add the id parameter as the last parameter to sortby by default, but only if it is not already passed as the last parameter.
@ -298,199 +297,22 @@ func getRawTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, op
})
}
// Since xorm does not use placeholders for order by, it is possible to expose this with sql injection if we're directly
// passing user input to the db.
// As a workaround to prevent this, we check for valid column names here prior to passing it to the db.
var orderby string
for i, param := range opts.sortby {
// Validate the params
if err := param.validate(); err != nil {
return nil, 0, 0, err
var searcher taskSearcher = &dbTaskSearcher{
s: s,
a: a,
hasFavoritesProject: hasFavoritesProject,
}
// Mysql sorts columns with null values before ones without null value.
// Because it does not have support for NULLS FIRST or NULLS LAST we work around this by
// first sorting for null (or not null) values and then the order we actually want to.
if db.Type() == schemas.MYSQL {
orderby += "`" + param.sortBy + "` IS NULL, "
}
orderby += "`" + param.sortBy + "` " + param.orderBy.String()
// Postgres and sqlite allow us to control how columns with null values are sorted.
// To make that consistent with the sort order we have and other dbms, we're adding a separate clause here.
if db.Type() == schemas.POSTGRES || db.Type() == schemas.SQLITE {
orderby += " NULLS LAST"
}
if (i + 1) < len(opts.sortby) {
orderby += ", "
if config.TypesenseEnabled.GetBool() {
searcher = &typesenseTaskSearcher{
s: s,
}
}
// Some filters need a special treatment since they are in a separate table
reminderFilters := []builder.Cond{}
assigneeFilters := []builder.Cond{}
labelFilters := []builder.Cond{}
projectFilters := []builder.Cond{}
var filters = make([]builder.Cond, 0, len(opts.filters))
// To still find tasks with nil values, we exclude 0s when comparing with >/< values.
for _, f := range opts.filters {
if f.field == "reminders" {
f.field = "reminder" // This is the name in the db
filter, err := getFilterCond(f, opts.filterIncludeNulls)
if err != nil {
return nil, 0, 0, err
}
reminderFilters = append(reminderFilters, filter)
continue
}
if f.field == "assignees" {
if f.comparator == taskFilterComparatorLike {
return nil, 0, 0, ErrInvalidTaskFilterValue{Field: f.field, Value: f.value}
}
f.field = "username"
filter, err := getFilterCond(f, opts.filterIncludeNulls)
if err != nil {
return nil, 0, 0, err
}
assigneeFilters = append(assigneeFilters, filter)
continue
}
if f.field == "labels" || f.field == "label_id" {
f.field = "label_id"
filter, err := getFilterCond(f, opts.filterIncludeNulls)
if err != nil {
return nil, 0, 0, err
}
labelFilters = append(labelFilters, filter)
continue
}
if f.field == "parent_project" || f.field == "parent_project_id" {
f.field = "parent_project_id"
filter, err := getFilterCond(f, opts.filterIncludeNulls)
if err != nil {
return nil, 0, 0, err
}
projectFilters = append(projectFilters, filter)
continue
}
filter, err := getFilterCond(f, opts.filterIncludeNulls)
if err != nil {
return nil, 0, 0, err
}
filters = append(filters, filter)
}
// Then return all tasks for that projects
var where builder.Cond
if opts.search != "" {
where = db.ILIKE("title", opts.search)
searchIndex := getTaskIndexFromSearchString(opts.search)
if searchIndex > 0 {
where = builder.Or(where, builder.Eq{"`index`": searchIndex})
}
}
var projectIDCond builder.Cond
var favoritesCond builder.Cond
if len(projectIDs) > 0 {
projectIDCond = builder.In("project_id", projectIDs)
}
if hasFavoritesProject {
// All favorite tasks for that user
favCond := builder.
Select("entity_id").
From("favorites").
Where(
builder.And(
builder.Eq{"user_id": a.GetID()},
builder.Eq{"kind": FavoriteKindTask},
))
favoritesCond = builder.In("id", favCond)
}
if len(reminderFilters) > 0 {
filters = append(filters, getFilterCondForSeparateTable("task_reminders", opts.filterConcat, reminderFilters))
}
if len(assigneeFilters) > 0 {
assigneeFilter := []builder.Cond{
builder.In("user_id",
builder.Select("id").
From("users").
Where(builder.Or(assigneeFilters...)),
)}
filters = append(filters, getFilterCondForSeparateTable("task_assignees", opts.filterConcat, assigneeFilter))
}
if len(labelFilters) > 0 {
filters = append(filters, getFilterCondForSeparateTable("label_tasks", opts.filterConcat, labelFilters))
}
if len(projectFilters) > 0 {
var filtercond builder.Cond
if opts.filterConcat == filterConcatOr {
filtercond = builder.Or(projectFilters...)
}
if opts.filterConcat == filterConcatAnd {
filtercond = builder.And(projectFilters...)
}
cond := builder.In(
"project_id",
builder.
Select("id").
From("projects").
Where(filtercond),
)
filters = append(filters, cond)
}
var filterCond builder.Cond
if len(filters) > 0 {
if opts.filterConcat == filterConcatOr {
filterCond = builder.Or(filters...)
}
if opts.filterConcat == filterConcatAnd {
filterCond = builder.And(filters...)
}
}
limit, start := getLimitFromPageIndex(opts.page, opts.perPage)
cond := builder.And(builder.Or(projectIDCond, favoritesCond), where, filterCond)
query := s.Where(cond)
if limit > 0 {
query = query.Limit(limit, start)
}
tasks = []*Task{}
err = query.OrderBy(orderby).Find(&tasks)
if err != nil {
return nil, 0, 0, err
}
queryCount := s.Where(cond)
totalItems, err = queryCount.
Count(&Task{})
if err != nil {
return nil, 0, 0, err
}
tasks, totalItems, err = searcher.Search(opts)
return tasks, len(tasks), totalItems, nil
}
func getTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, opts *taskOptions) (tasks []*Task, resultCount int, totalItems int64, err error) {
func getTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, opts *taskSearchOptions) (tasks []*Task, resultCount int, totalItems int64, err error) {
tasks, resultCount, totalItems, err = getRawTasksForProjects(s, projects, a, opts)
if err != nil {

2
pkg/models/typesense.go
View File

@ -248,7 +248,7 @@ type typesenseTask struct {
Reminders interface{} `json:"reminders"`
Assignees interface{} `json:"assignees"`
Labels interface{} `json:"labels"`
//RelatedTasks interface{} `json:"related_tasks"`
//RelatedTasks interface{} `json:"related_tasks"` // TODO
Attachments interface{} `json:"attachments"`
Comments interface{} `json:"comments"`
}