From 307ffe11c4ce31df98db3164cc7d21412839d886 Mon Sep 17 00:00:00 2001 From: kolaente Date: Thu, 9 Nov 2023 13:34:31 +0100 Subject: [PATCH 01/62] feat(filters): very basic filter parsing --- go.sum | 6 ++++++ pkg/models/task_collection.go | 13 ++++++++----- pkg/models/task_collection_filter.go | 20 +++++++++++++++----- pkg/models/task_collection_test.go | 5 +++++ pkg/models/tasks.go | 1 + 5 files changed, 35 insertions(+), 10 deletions(-) diff --git a/go.sum b/go.sum index 1fed95db1..eecaface0 100644 --- a/go.sum +++ b/go.sum @@ -109,6 +109,12 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/ganigeorgiev/fexpr v0.3.0 h1:RwSyJBME+g/XdzlUW0paH/4VXhLHPg+rErtLeC7K8Ew= +github.com/ganigeorgiev/fexpr v0.3.0/go.mod h1:RyGiGqmeXhEQ6+mlGdnUleLHgtzzu/VGO2WtJkF5drE= +github.com/getsentry/sentry-go v0.25.0 h1:q6Eo+hS+yoJlTO3uu/azhQadsD8V+jQn2D8VvX1eOyI= +github.com/getsentry/sentry-go v0.25.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/getsentry/sentry-go v0.26.0 h1:IX3++sF6/4B5JcevhdZfdKIHfyvMmAq/UnqcyT2H6mA= +github.com/getsentry/sentry-go v0.26.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= diff --git a/pkg/models/task_collection.go b/pkg/models/task_collection.go index 25b78e463..99fc58431 100644 --- a/pkg/models/task_collection.go +++ b/pkg/models/task_collection.go @@ -33,17 +33,20 @@ type TaskCollection struct { OrderBy []string `query:"order_by" json:"order_by"` OrderByArr []string `query:"order_by[]" json:"-"` - // The field name of the field to filter by + // Deprecated: The field name of the field to filter by FilterBy []string `query:"filter_by" json:"filter_by"` FilterByArr []string `query:"filter_by[]" json:"-"` - // The value of the field name to filter by + // Deprecated: The value of the field name to filter by FilterValue []string `query:"filter_value" json:"filter_value"` FilterValueArr []string `query:"filter_value[]" json:"-"` - // The comparator for field and value + // Deprecated: The comparator for field and value FilterComparator []string `query:"filter_comparator" json:"filter_comparator"` FilterComparatorArr []string `query:"filter_comparator[]" json:"-"` - // The way all filter conditions are concatenated together, can be either "and" or "or"., + // Deprecated: The way all filter conditions are concatenated together, can be either "and" or "or"., FilterConcat string `query:"filter_concat" json:"filter_concat"` + + Filter string `query:"filter" json:"filter"` + // If set to true, the result will also include null values FilterIncludeNulls bool `query:"filter_include_nulls" json:"filter_include_nulls"` @@ -110,8 +113,8 @@ func getTaskFilterOptsFromCollection(tf *TaskCollection) (opts *taskSearchOption opts = &taskSearchOptions{ sortby: sort, - filterConcat: taskFilterConcatinator(tf.FilterConcat), filterIncludeNulls: tf.FilterIncludeNulls, + filter: tf.Filter, } opts.filters, err = getTaskFiltersByCollections(tf) diff --git a/pkg/models/task_collection_filter.go b/pkg/models/task_collection_filter.go index c9db96210..ab87b3902 100644 --- a/pkg/models/task_collection_filter.go +++ b/pkg/models/task_collection_filter.go @@ -18,6 +18,7 @@ package models import ( "fmt" + "github.com/ganigeorgiev/fexpr" "reflect" "strconv" "strings" @@ -108,11 +109,20 @@ func getTaskFiltersByCollections(c *TaskCollection) (filters []*taskFilter, err } } - filters = make([]*taskFilter, 0, len(c.FilterBy)) - for i, f := range c.FilterBy { - filter := &taskFilter{ - field: f, - comparator: taskFilterComparatorEquals, + parsedFilter, err := fexpr.Parse(c.Filter) + if err != nil { + return nil, err + } + + filters = make([]*taskFilter, 0, len(parsedFilter)) + for i, f := range parsedFilter { + + filter := &taskFilter{} + switch v := f.Item.(type) { + case fexpr.Expr: + filter.field = v.Left.Literal + filter.comparator = v.Op + filter.value = v.Right.Literal // TODO: nesting } if len(c.FilterComparator) > i { diff --git a/pkg/models/task_collection_test.go b/pkg/models/task_collection_test.go index d87e9da0b..77830979a 100644 --- a/pkg/models/task_collection_test.go +++ b/pkg/models/task_collection_test.go @@ -680,6 +680,8 @@ func TestTaskCollection_ReadAll(t *testing.T) { FilterComparator []string FilterIncludeNulls bool + Filter string + CRUDable web.CRUDable Rights web.Rights } @@ -795,6 +797,7 @@ func TestTaskCollection_ReadAll(t *testing.T) { FilterBy: []string{"start_date", "end_date"}, FilterValue: []string{"2018-12-11T03:46:40+00:00", "2018-12-13T11:20:01+00:00"}, FilterComparator: []string{"greater", "less"}, + Filter: "start_date>'2018-12-11T03:46:40+00:00' || end_date<'2018-12-13T11:20:01+00:00'", }, args: defaultArgs, want: []*Task{ @@ -1339,6 +1342,8 @@ func TestTaskCollection_ReadAll(t *testing.T) { FilterComparator: tt.fields.FilterComparator, FilterIncludeNulls: tt.fields.FilterIncludeNulls, + Filter: tt.fields.Filter, + CRUDable: tt.fields.CRUDable, Rights: tt.fields.Rights, } diff --git a/pkg/models/tasks.go b/pkg/models/tasks.go index be21e9f00..7b0f2a36c 100644 --- a/pkg/models/tasks.go +++ b/pkg/models/tasks.go @@ -174,6 +174,7 @@ type taskSearchOptions struct { filters []*taskFilter filterConcat taskFilterConcatinator filterIncludeNulls bool + filter string projectIDs []int64 } From de320aac723d9bf576e9ba390fd544a108ec84fe Mon Sep 17 00:00:00 2001 From: kolaente Date: Thu, 9 Nov 2023 22:31:06 +0100 Subject: [PATCH 02/62] feat(filters): basic text filter works now --- pkg/models/task_collection_filter.go | 65 +++++++++++++++------- pkg/models/task_collection_test.go | 8 +-- pkg/models/task_search.go | 80 ++++++++++------------------ pkg/models/tasks.go | 27 ++++------ 4 files changed, 87 insertions(+), 93 deletions(-) diff --git a/pkg/models/task_collection_filter.go b/pkg/models/task_collection_filter.go index ab87b3902..7e7da7836 100644 --- a/pkg/models/task_collection_filter.go +++ b/pkg/models/task_collection_filter.go @@ -55,6 +55,7 @@ type taskFilter struct { value interface{} // Needs to be an interface to be able to hold the field's native value comparator taskFilterComparator isNumeric bool + join taskFilterConcatinator } func parseTimeFromUserInput(timeString string) (value time.Time, err error) { @@ -103,11 +104,11 @@ func getTaskFiltersByCollections(c *TaskCollection) (filters []*taskFilter, err c.FilterComparator = append(c.FilterComparator, c.FilterComparatorArr...) } - if c.FilterConcat != "" && c.FilterConcat != filterConcatAnd && c.FilterConcat != filterConcatOr { - return nil, ErrInvalidTaskFilterConcatinator{ - Concatinator: taskFilterConcatinator(c.FilterConcat), - } - } + //if c.FilterConcat != "" && c.FilterConcat != filterConcatAnd && c.FilterConcat != filterConcatOr { + // return nil, ErrInvalidTaskFilterConcatinator{ + // Concatinator: taskFilterConcatinator(c.FilterConcat), + // } + //} parsedFilter, err := fexpr.Parse(c.Filter) if err != nil { @@ -115,18 +116,21 @@ func getTaskFiltersByCollections(c *TaskCollection) (filters []*taskFilter, err } filters = make([]*taskFilter, 0, len(parsedFilter)) - for i, f := range parsedFilter { + for _, f := range parsedFilter { - filter := &taskFilter{} + filter := &taskFilter{ + join: filterConcatAnd, + } + if f.Join == fexpr.JoinOr { + filter.join = filterConcatOr + } + + var value string switch v := f.Item.(type) { case fexpr.Expr: filter.field = v.Left.Literal - filter.comparator = v.Op - filter.value = v.Right.Literal // TODO: nesting - } - - if len(c.FilterComparator) > i { - filter.comparator, err = getFilterComparatorFromString(c.FilterComparator[i]) + value = v.Right.Literal // TODO: nesting + filter.comparator, err = getFilterComparatorFromOp(v.Op) if err != nil { return } @@ -139,13 +143,11 @@ func getTaskFiltersByCollections(c *TaskCollection) (filters []*taskFilter, err // Cast the field value to its native type var reflectValue *reflect.StructField - if len(c.FilterValue) > i { - reflectValue, filter.value, err = getNativeValueForTaskField(filter.field, filter.comparator, c.FilterValue[i]) - if err != nil { - return nil, ErrInvalidTaskFilterValue{ - Value: filter.field, - Field: c.FilterValue[i], - } + reflectValue, filter.value, err = getNativeValueForTaskField(filter.field, filter.comparator, value) + if err != nil { + return nil, ErrInvalidTaskFilterValue{ + Value: filter.field, + Field: value, } } if reflectValue != nil { @@ -200,6 +202,29 @@ func getFilterComparatorFromString(comparator string) (taskFilterComparator, err } } +func getFilterComparatorFromOp(op fexpr.SignOp) (taskFilterComparator, error) { + switch op { + case fexpr.SignEq: + return taskFilterComparatorEquals, nil + case fexpr.SignGt: + return taskFilterComparatorGreater, nil + case fexpr.SignGte: + return taskFilterComparatorGreateEquals, nil + case fexpr.SignLt: + return taskFilterComparatorLess, nil + case fexpr.SignLte: + return taskFilterComparatorLessEquals, nil + case fexpr.SignNeq: + return taskFilterComparatorNotEquals, nil + case fexpr.SignLike: + return taskFilterComparatorLike, nil + case "in": + return taskFilterComparatorIn, nil + default: + return taskFilterComparatorInvalid, ErrInvalidTaskFilterComparator{Comparator: taskFilterComparator(op)} + } +} + func getValueForField(field reflect.StructField, rawValue string) (value interface{}, err error) { switch field.Type.Kind() { case reflect.Int64: diff --git a/pkg/models/task_collection_test.go b/pkg/models/task_collection_test.go index 77830979a..52d9117c8 100644 --- a/pkg/models/task_collection_test.go +++ b/pkg/models/task_collection_test.go @@ -794,10 +794,10 @@ func TestTaskCollection_ReadAll(t *testing.T) { { name: "ReadAll Tasks with range", fields: fields{ - FilterBy: []string{"start_date", "end_date"}, - FilterValue: []string{"2018-12-11T03:46:40+00:00", "2018-12-13T11:20:01+00:00"}, - FilterComparator: []string{"greater", "less"}, - Filter: "start_date>'2018-12-11T03:46:40+00:00' || end_date<'2018-12-13T11:20:01+00:00'", + //FilterBy: []string{"start_date", "end_date"}, + //FilterValue: []string{"2018-12-11T03:46:40+00:00", "2018-12-13T11:20:01+00:00"}, + //FilterComparator: []string{"greater", "less"}, + Filter: "start_date > '2018-12-11T03:46:40+00:00' || end_date < '2018-12-13T11:20:01+00:00'", }, args: defaultArgs, want: []*Task{ diff --git a/pkg/models/task_search.go b/pkg/models/task_search.go index 8a3601f3f..f1a00f4e4 100644 --- a/pkg/models/task_search.go +++ b/pkg/models/task_search.go @@ -84,12 +84,6 @@ func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCo return nil, 0, err } - // Some filters need a special treatment since they are in a separate table - reminderFilters := []builder.Cond{} - assigneeFilters := []builder.Cond{} - labelFilters := []builder.Cond{} - projectFilters := []builder.Cond{} - var filters = make([]builder.Cond, 0, len(opts.filters)) // To still find tasks with nil values, we exclude 0s when comparing with >/< values. for _, f := range opts.filters { @@ -104,7 +98,7 @@ func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCo if err != nil { return nil, totalCount, err } - reminderFilters = append(reminderFilters, filter) + filters = append(filters, getFilterCondForSeparateTable("task_reminders", filter)) continue } @@ -122,7 +116,13 @@ func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCo if err != nil { return nil, totalCount, err } - assigneeFilters = append(assigneeFilters, filter) + + assigneeFilter := builder.In("user_id", + builder.Select("id"). + From("users"). + Where(filter), + ) + filters = append(filters, getFilterCondForSeparateTable("task_assignees", assigneeFilter)) continue } @@ -137,7 +137,8 @@ func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCo if err != nil { return nil, totalCount, err } - labelFilters = append(labelFilters, filter) + + filters = append(filters, getFilterCondForSeparateTable("label_tasks", filter)) continue } @@ -152,7 +153,15 @@ func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCo if err != nil { return nil, totalCount, err } - projectFilters = append(projectFilters, filter) + + cond := builder.In( + "project_id", + builder. + Select("id"). + From("projects"). + Where(filter), + ) + filters = append(filters, cond) continue } @@ -199,50 +208,17 @@ func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCo favoritesCond = builder.In("id", favCond) } - if len(reminderFilters) > 0 { - filters = append(filters, getFilterCondForSeparateTable("task_reminders", opts.filterConcat, reminderFilters)) - } - - if len(assigneeFilters) > 0 { - assigneeFilter := []builder.Cond{ - builder.In("user_id", - builder.Select("id"). - From("users"). - Where(builder.Or(assigneeFilters...)), - )} - filters = append(filters, getFilterCondForSeparateTable("task_assignees", opts.filterConcat, assigneeFilter)) - } - - if len(labelFilters) > 0 { - filters = append(filters, getFilterCondForSeparateTable("label_tasks", opts.filterConcat, labelFilters)) - } - - if len(projectFilters) > 0 { - var filtercond builder.Cond - if opts.filterConcat == filterConcatOr { - filtercond = builder.Or(projectFilters...) - } - if opts.filterConcat == filterConcatAnd { - filtercond = builder.And(projectFilters...) - } - - cond := builder.In( - "project_id", - builder. - Select("id"). - From("projects"). - Where(filtercond), - ) - filters = append(filters, cond) - } - var filterCond builder.Cond if len(filters) > 0 { - if opts.filterConcat == filterConcatOr { - filterCond = builder.Or(filters...) - } - if opts.filterConcat == filterConcatAnd { - filterCond = builder.And(filters...) + for i, f := range filters { + if len(filters) > i+1 { + switch opts.filters[i].join { + case filterConcatOr: + filterCond = builder.Or(filterCond, f, filters[i+1]) + case filterConcatAnd: + filterCond = builder.And(filterCond, f, filters[i+1]) + } + } } } diff --git a/pkg/models/tasks.go b/pkg/models/tasks.go index 7b0f2a36c..613e41246 100644 --- a/pkg/models/tasks.go +++ b/pkg/models/tasks.go @@ -162,16 +162,17 @@ func (t *Task) GetFrontendURL() string { type taskFilterConcatinator string const ( - filterConcatAnd = "and" - filterConcatOr = "or" + filterConcatAnd taskFilterConcatinator = "and" + filterConcatOr taskFilterConcatinator = "or" ) type taskSearchOptions struct { - search string - page int - perPage int - sortby []*sortParam - filters []*taskFilter + search string + page int + perPage int + sortby []*sortParam + filters []*taskFilter + // deprecated: concat should live in filters directly filterConcat taskFilterConcatinator filterIncludeNulls bool filter string @@ -239,21 +240,13 @@ func getFilterCond(f *taskFilter, includeNulls bool) (cond builder.Cond, err err return } -func getFilterCondForSeparateTable(table string, concat taskFilterConcatinator, conds []builder.Cond) builder.Cond { - var filtercond builder.Cond - if concat == filterConcatOr { - filtercond = builder.Or(conds...) - } - if concat == filterConcatAnd { - filtercond = builder.And(conds...) - } - +func getFilterCondForSeparateTable(table string, cond builder.Cond) builder.Cond { return builder.In( "id", builder. Select("task_id"). From(table). - Where(filtercond), + Where(cond), ) } From c1e137d8ee718a412d5f616b0970c0ac2f55b588 Mon Sep 17 00:00:00 2001 From: kolaente Date: Tue, 21 Nov 2023 18:22:56 +0100 Subject: [PATCH 03/62] fix(filter): make sure single filter condition works --- pkg/models/task_search.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/pkg/models/task_search.go b/pkg/models/task_search.go index f1a00f4e4..f936a86cf 100644 --- a/pkg/models/task_search.go +++ b/pkg/models/task_search.go @@ -210,13 +210,17 @@ func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCo var filterCond builder.Cond if len(filters) > 0 { - for i, f := range filters { - if len(filters) > i+1 { - switch opts.filters[i].join { - case filterConcatOr: - filterCond = builder.Or(filterCond, f, filters[i+1]) - case filterConcatAnd: - filterCond = builder.And(filterCond, f, filters[i+1]) + if len(filters) == 1 { + filterCond = filters[0] + } else { + for i, f := range filters { + if len(filters) > i+1 { + switch opts.filters[i+1].join { + case filterConcatOr: + filterCond = builder.Or(filterCond, f, filters[i+1]) + case filterConcatAnd: + filterCond = builder.And(filterCond, f, filters[i+1]) + } } } } From 9f73e2c5f904b89743125bccab661c03f226de94 Mon Sep 17 00:00:00 2001 From: kolaente Date: Tue, 21 Nov 2023 18:23:17 +0100 Subject: [PATCH 04/62] fix(filter): don't crash on empty filter --- pkg/models/task_collection_filter.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/models/task_collection_filter.go b/pkg/models/task_collection_filter.go index 7e7da7836..f0019fd6e 100644 --- a/pkg/models/task_collection_filter.go +++ b/pkg/models/task_collection_filter.go @@ -92,6 +92,10 @@ func parseTimeFromUserInput(timeString string) (value time.Time, err error) { func getTaskFiltersByCollections(c *TaskCollection) (filters []*taskFilter, err error) { + if c.Filter == "" { + return + } + if len(c.FilterByArr) > 0 { c.FilterBy = append(c.FilterBy, c.FilterByArr...) } From 3fc4aaa2a141efaf641445878337f9bbb6f204d0 Mon Sep 17 00:00:00 2001 From: kolaente Date: Tue, 21 Nov 2023 18:23:30 +0100 Subject: [PATCH 05/62] fix(filter): allow filtering on "in" condition --- pkg/models/task_collection_filter.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/models/task_collection_filter.go b/pkg/models/task_collection_filter.go index f0019fd6e..6eabd3fee 100644 --- a/pkg/models/task_collection_filter.go +++ b/pkg/models/task_collection_filter.go @@ -222,6 +222,8 @@ func getFilterComparatorFromOp(op fexpr.SignOp) (taskFilterComparator, error) { return taskFilterComparatorNotEquals, nil case fexpr.SignLike: return taskFilterComparatorLike, nil + case fexpr.SignAnyEq: + fallthrough case "in": return taskFilterComparatorIn, nil default: From 764bc15d49bef7ad938c8a3c8e2d2dd4b4760376 Mon Sep 17 00:00:00 2001 From: kolaente Date: Tue, 21 Nov 2023 18:35:33 +0100 Subject: [PATCH 06/62] fix(filter): allow filtering for "project" --- pkg/models/task_collection_filter.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/models/task_collection_filter.go b/pkg/models/task_collection_filter.go index 6eabd3fee..d285c60e2 100644 --- a/pkg/models/task_collection_filter.go +++ b/pkg/models/task_collection_filter.go @@ -147,6 +147,9 @@ func getTaskFiltersByCollections(c *TaskCollection) (filters []*taskFilter, err // Cast the field value to its native type var reflectValue *reflect.StructField + if filter.field == "project" { + filter.field = "project_id" + } reflectValue, filter.value, err = getNativeValueForTaskField(filter.field, filter.comparator, value) if err != nil { return nil, ErrInvalidTaskFilterValue{ From 9624cc9e9709b7d3a8c8abe1cf00ecce5797e564 Mon Sep 17 00:00:00 2001 From: kolaente Date: Tue, 21 Nov 2023 18:37:07 +0100 Subject: [PATCH 07/62] fix(filter): translate all tests --- pkg/db/fixtures/saved_filters.yml | 2 +- pkg/models/task_collection_test.go | 130 ++++++++++++++--------------- 2 files changed, 64 insertions(+), 68 deletions(-) diff --git a/pkg/db/fixtures/saved_filters.yml b/pkg/db/fixtures/saved_filters.yml index 844ceb1a4..43cc486ca 100644 --- a/pkg/db/fixtures/saved_filters.yml +++ b/pkg/db/fixtures/saved_filters.yml @@ -1,5 +1,5 @@ - id: 1 - filters: '{"sort_by":null,"order_by":null,"filter_by":["start_date","end_date","due_date"],"filter_value":["2018-12-11T03:46:40+00:00","2018-12-13T11:20:01+00:00","2018-11-29T14:00:00+00:00"],"filter_comparator":["greater","less","greater"],"filter_concat":"","filter_include_nulls":false}' + filters: '{"sort_by":null,"order_by":null,"filter":"start_date > \u00272018-12-11T03:46:40+00:00\u0027 || end_date < \u00272018-12-13T11:20:01+00:00\u0027 || due_date > \u00272018-11-29T14:00:00+00:00\u0027","filter_include_nulls":false}' title: testfilter1 owner_id: 1 updated: 2020-09-08 15:13:12 diff --git a/pkg/models/task_collection_test.go b/pkg/models/task_collection_test.go index 52d9117c8..a3e72af6b 100644 --- a/pkg/models/task_collection_test.go +++ b/pkg/models/task_collection_test.go @@ -29,6 +29,8 @@ import ( "gopkg.in/d4l3k/messagediff.v1" ) +// To only run a selected tests: ^\QTestTaskCollection_ReadAll\E$/^\QReadAll_Tasks_with_range\E$ + func TestTaskCollection_ReadAll(t *testing.T) { // Dummy users user1 := &user.User{ @@ -794,9 +796,6 @@ func TestTaskCollection_ReadAll(t *testing.T) { { name: "ReadAll Tasks with range", fields: fields{ - //FilterBy: []string{"start_date", "end_date"}, - //FilterValue: []string{"2018-12-11T03:46:40+00:00", "2018-12-13T11:20:01+00:00"}, - //FilterComparator: []string{"greater", "less"}, Filter: "start_date > '2018-12-11T03:46:40+00:00' || end_date < '2018-12-13T11:20:01+00:00'", }, args: defaultArgs, @@ -810,9 +809,7 @@ func TestTaskCollection_ReadAll(t *testing.T) { { name: "ReadAll Tasks with different range", fields: fields{ - FilterBy: []string{"start_date", "end_date"}, - FilterValue: []string{"2018-12-13T11:20:00+00:00", "2018-12-16T22:40:00+00:00"}, - FilterComparator: []string{"greater", "less"}, + Filter: "start_date > '2018-12-13T11:20:00+00:00' || end_date < '2018-12-16T22:40:00+00:00'", }, args: defaultArgs, want: []*Task{ @@ -824,9 +821,7 @@ func TestTaskCollection_ReadAll(t *testing.T) { { name: "ReadAll Tasks with range with start date only", fields: fields{ - FilterBy: []string{"start_date"}, - FilterValue: []string{"2018-12-12T07:33:20+00:00"}, - FilterComparator: []string{"greater"}, + Filter: "start_date > '2018-12-12T07:33:20+00:00'", }, args: defaultArgs, want: []*Task{}, @@ -835,9 +830,7 @@ func TestTaskCollection_ReadAll(t *testing.T) { { name: "ReadAll Tasks with range with start date only and greater equals", fields: fields{ - FilterBy: []string{"start_date"}, - FilterValue: []string{"2018-12-12T07:33:20+00:00"}, - FilterComparator: []string{"greater_equals"}, + Filter: "start_date >= '2018-12-12T07:33:20+00:00'", }, args: defaultArgs, want: []*Task{ @@ -849,9 +842,7 @@ func TestTaskCollection_ReadAll(t *testing.T) { { name: "undone tasks only", fields: fields{ - FilterBy: []string{"done"}, - FilterValue: []string{"false"}, - FilterComparator: []string{"equals"}, + Filter: "done = false", }, args: defaultArgs, want: []*Task{ @@ -895,9 +886,7 @@ func TestTaskCollection_ReadAll(t *testing.T) { { name: "done tasks only", fields: fields{ - FilterBy: []string{"done"}, - FilterValue: []string{"true"}, - FilterComparator: []string{"equals"}, + Filter: "done = true", }, args: defaultArgs, want: []*Task{ @@ -908,9 +897,7 @@ func TestTaskCollection_ReadAll(t *testing.T) { { name: "done tasks only - not equals done", fields: fields{ - FilterBy: []string{"done"}, - FilterValue: []string{"false"}, - FilterComparator: []string{"not_equals"}, + Filter: "done != false", }, args: defaultArgs, want: []*Task{ @@ -921,10 +908,8 @@ func TestTaskCollection_ReadAll(t *testing.T) { { name: "range with nulls", fields: fields{ - FilterBy: []string{"start_date", "end_date"}, - FilterValue: []string{"2018-12-11T03:46:40+00:00", "2018-12-13T11:20:01+00:00"}, - FilterComparator: []string{"greater", "less"}, FilterIncludeNulls: true, + Filter: "start_date > '2018-12-11T03:46:40+00:00' || end_date < '2018-12-13T11:20:01+00:00'", }, args: defaultArgs, want: []*Task{ @@ -979,9 +964,26 @@ func TestTaskCollection_ReadAll(t *testing.T) { { name: "filtered with like", fields: fields{ - FilterBy: []string{"title"}, - FilterValue: []string{"with"}, - FilterComparator: []string{"like"}, + Filter: "title ~ with", + }, + args: defaultArgs, + want: []*Task{ + task7, + task8, + task9, + task27, + task28, + task29, + task30, + task31, + task33, + }, + wantErr: false, + }, + { + name: "filtered with like and '", + fields: fields{ + Filter: "title ~ 'with'", }, args: defaultArgs, want: []*Task{ @@ -1000,9 +1002,7 @@ func TestTaskCollection_ReadAll(t *testing.T) { { name: "filtered reminder dates", fields: fields{ - FilterBy: []string{"reminders", "reminders"}, - FilterValue: []string{"2018-10-01T00:00:00+00:00", "2018-12-10T00:00:00+00:00"}, - FilterComparator: []string{"greater", "less"}, + Filter: "reminders > '2018-10-01T00:00:00+00:00' && reminders < '2018-12-10T00:00:00+00:00'", }, args: defaultArgs, want: []*Task{ @@ -1014,9 +1014,7 @@ func TestTaskCollection_ReadAll(t *testing.T) { { name: "filter in", fields: fields{ - FilterBy: []string{"id"}, - FilterValue: []string{"1,2,34"}, // Task 34 is forbidden for user 1 - FilterComparator: []string{"in"}, + Filter: "id ?= '1,2,34'", }, args: defaultArgs, want: []*Task{ @@ -1028,9 +1026,7 @@ func TestTaskCollection_ReadAll(t *testing.T) { { name: "filter assignees by username", fields: fields{ - FilterBy: []string{"assignees"}, - FilterValue: []string{"user1"}, - FilterComparator: []string{"equals"}, + Filter: "assignees = 'user1'", }, args: defaultArgs, want: []*Task{ @@ -1041,9 +1037,7 @@ func TestTaskCollection_ReadAll(t *testing.T) { { name: "filter assignees by username with users field name", fields: fields{ - FilterBy: []string{"users"}, - FilterValue: []string{"user1"}, - FilterComparator: []string{"equals"}, + Filter: "users = 'user1'", }, args: defaultArgs, want: nil, @@ -1052,9 +1046,7 @@ func TestTaskCollection_ReadAll(t *testing.T) { { name: "filter assignees by username with user_id field name", fields: fields{ - FilterBy: []string{"user_id"}, - FilterValue: []string{"user1"}, - FilterComparator: []string{"equals"}, + Filter: "user_id = 'user1'", }, args: defaultArgs, want: nil, @@ -1063,9 +1055,7 @@ func TestTaskCollection_ReadAll(t *testing.T) { { name: "filter assignees by multiple username", fields: fields{ - FilterBy: []string{"assignees", "assignees"}, - FilterValue: []string{"user1", "user2"}, - FilterComparator: []string{"equals", "equals"}, + Filter: "assignees = 'user1' || assignees = 'user2'", }, args: defaultArgs, want: []*Task{ @@ -1077,9 +1067,7 @@ func TestTaskCollection_ReadAll(t *testing.T) { { name: "filter assignees by numbers", fields: fields{ - FilterBy: []string{"assignees"}, - FilterValue: []string{"1"}, - FilterComparator: []string{"equals"}, + Filter: "assignees = 1", }, args: defaultArgs, want: []*Task{}, @@ -1088,9 +1076,7 @@ func TestTaskCollection_ReadAll(t *testing.T) { { name: "filter assignees by name with like", fields: fields{ - FilterBy: []string{"assignees"}, - FilterValue: []string{"user"}, - FilterComparator: []string{"like"}, + Filter: "assignees ~ 'user'", }, args: defaultArgs, want: []*Task{}, @@ -1099,9 +1085,7 @@ func TestTaskCollection_ReadAll(t *testing.T) { { name: "filter assignees in by id", fields: fields{ - FilterBy: []string{"assignees"}, - FilterValue: []string{"1,2"}, - FilterComparator: []string{"in"}, + Filter: "assignees ?= '1,2'", }, args: defaultArgs, want: []*Task{}, @@ -1110,9 +1094,7 @@ func TestTaskCollection_ReadAll(t *testing.T) { { name: "filter assignees in by username", fields: fields{ - FilterBy: []string{"assignees"}, - FilterValue: []string{"user1,user2"}, - FilterComparator: []string{"in"}, + Filter: "assignees ?= 'user1,user2'", }, args: defaultArgs, want: []*Task{ @@ -1124,9 +1106,7 @@ func TestTaskCollection_ReadAll(t *testing.T) { { name: "filter labels", fields: fields{ - FilterBy: []string{"labels"}, - FilterValue: []string{"4"}, - FilterComparator: []string{"equals"}, + Filter: "labels = 4", }, args: defaultArgs, want: []*Task{ @@ -1137,11 +1117,9 @@ func TestTaskCollection_ReadAll(t *testing.T) { wantErr: false, }, { - name: "filter project", + name: "filter project_id", fields: fields{ - FilterBy: []string{"project_id"}, - FilterValue: []string{"6"}, - FilterComparator: []string{"equals"}, + Filter: "project_id = 6", }, args: defaultArgs, want: []*Task{ @@ -1149,13 +1127,31 @@ func TestTaskCollection_ReadAll(t *testing.T) { }, wantErr: false, }, + { + name: "filter project", + fields: fields{ + Filter: "project = 6", + }, + args: defaultArgs, + want: []*Task{ + task15, + }, + wantErr: false, + }, + { + name: "filter project forbidden", + fields: fields{ + Filter: "project_id = 20", // user1 has no access to project 20 + }, + args: defaultArgs, + want: []*Task{}, + wantErr: false, + }, // TODO filter parent project? { name: "filter by index", fields: fields{ - FilterBy: []string{"index"}, - FilterValue: []string{"5"}, - FilterComparator: []string{"equals"}, + Filter: "index = 5", }, args: defaultArgs, want: []*Task{ From e43349618b3d35e1ec71f4d0a1285e1594bc5bad Mon Sep 17 00:00:00 2001 From: kolaente Date: Tue, 21 Nov 2023 18:40:13 +0100 Subject: [PATCH 08/62] feat(filter): more tests --- pkg/models/task_collection_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/models/task_collection_test.go b/pkg/models/task_collection_test.go index a3e72af6b..ad77dba7a 100644 --- a/pkg/models/task_collection_test.go +++ b/pkg/models/task_collection_test.go @@ -827,6 +827,18 @@ func TestTaskCollection_ReadAll(t *testing.T) { want: []*Task{}, wantErr: false, }, + { + name: "ReadAll Tasks with range with start date only between", + fields: fields{ + Filter: "start_date > '2018-12-12T00:00:00+00:00' && start_date < '2018-12-13T00:00:00+00:00'", + }, + args: defaultArgs, + want: []*Task{ + task7, + task9, + }, + wantErr: false, + }, { name: "ReadAll Tasks with range with start date only and greater equals", fields: fields{ From 76ed2cff5f9501ff5fbaec3acccf0cd5e7b2191b Mon Sep 17 00:00:00 2001 From: kolaente Date: Tue, 21 Nov 2023 19:18:01 +0100 Subject: [PATCH 09/62] feat(filter): nesting --- pkg/models/task_collection_filter.go | 94 ++++++++++++++---------- pkg/models/task_collection_test.go | 51 ++++++++++++- pkg/models/task_search.go | 105 ++++++++++++++++----------- 3 files changed, 167 insertions(+), 83 deletions(-) diff --git a/pkg/models/task_collection_filter.go b/pkg/models/task_collection_filter.go index d285c60e2..a5017edd9 100644 --- a/pkg/models/task_collection_filter.go +++ b/pkg/models/task_collection_filter.go @@ -90,6 +90,60 @@ func parseTimeFromUserInput(timeString string) (value time.Time, err error) { return value.In(config.GetTimeZone()), err } +func parseFilterFromExpression(f fexpr.ExprGroup) (filter *taskFilter, err error) { + filter = &taskFilter{ + join: filterConcatAnd, + } + if f.Join == fexpr.JoinOr { + filter.join = filterConcatOr + } + + var value string + switch v := f.Item.(type) { + case fexpr.Expr: + filter.field = v.Left.Literal + value = v.Right.Literal + filter.comparator, err = getFilterComparatorFromOp(v.Op) + if err != nil { + return + } + case []fexpr.ExprGroup: + values := make([]*taskFilter, 0, len(v)) + for _, expression := range v { + subfilter, err := parseFilterFromExpression(expression) + if err != nil { + return nil, err + } + values = append(values, subfilter) + } + filter.value = values + return + } + + err = validateTaskFieldComparator(filter.comparator) + if err != nil { + return + } + + // Cast the field value to its native type + var reflectValue *reflect.StructField + if filter.field == "project" { + filter.field = "project_id" + } + reflectValue, filter.value, err = getNativeValueForTaskField(filter.field, filter.comparator, value) + if err != nil { + return nil, ErrInvalidTaskFilterValue{ + Value: filter.field, + Field: value, + } + } + if reflectValue != nil { + filter.isNumeric = reflectValue.Type.Kind() == reflect.Int64 + } + + return filter, nil +} + func getTaskFiltersByCollections(c *TaskCollection) (filters []*taskFilter, err error) { if c.Filter == "" { @@ -121,46 +175,10 @@ func getTaskFiltersByCollections(c *TaskCollection) (filters []*taskFilter, err filters = make([]*taskFilter, 0, len(parsedFilter)) for _, f := range parsedFilter { - - filter := &taskFilter{ - join: filterConcatAnd, - } - if f.Join == fexpr.JoinOr { - filter.join = filterConcatOr - } - - var value string - switch v := f.Item.(type) { - case fexpr.Expr: - filter.field = v.Left.Literal - value = v.Right.Literal // TODO: nesting - filter.comparator, err = getFilterComparatorFromOp(v.Op) - if err != nil { - return - } - } - - err = validateTaskFieldComparator(filter.comparator) + filter, err := parseFilterFromExpression(f) if err != nil { - return + return nil, err } - - // Cast the field value to its native type - var reflectValue *reflect.StructField - if filter.field == "project" { - filter.field = "project_id" - } - reflectValue, filter.value, err = getNativeValueForTaskField(filter.field, filter.comparator, value) - if err != nil { - return nil, ErrInvalidTaskFilterValue{ - Value: filter.field, - Field: value, - } - } - if reflectValue != nil { - filter.isNumeric = reflectValue.Type.Kind() == reflect.Int64 - } - filters = append(filters, filter) } diff --git a/pkg/models/task_collection_test.go b/pkg/models/task_collection_test.go index ad77dba7a..494135fc8 100644 --- a/pkg/models/task_collection_test.go +++ b/pkg/models/task_collection_test.go @@ -851,6 +851,19 @@ func TestTaskCollection_ReadAll(t *testing.T) { }, wantErr: false, }, + { + name: "range and nesting", + fields: fields{ + Filter: "(start_date > '2018-12-12T00:00:00+00:00' && start_date < '2018-12-13T00:00:00+00:00') || end_date > '2018-12-13T00:00:00+00:00'", + }, + args: defaultArgs, + want: []*Task{ + task7, + task8, + task9, + }, + wantErr: false, + }, { name: "undone tasks only", fields: fields{ @@ -1090,8 +1103,42 @@ func TestTaskCollection_ReadAll(t *testing.T) { fields: fields{ Filter: "assignees ~ 'user'", }, - args: defaultArgs, - want: []*Task{}, + args: defaultArgs, + want: []*Task{ + // Same as without any filter since the filter is ignored + task1, + task2, + task3, + task4, + task5, + task6, + task7, + task8, + task9, + task10, + task11, + task12, + task15, + task16, + task17, + task18, + task19, + task20, + task21, + task22, + task23, + task24, + task25, + task26, + task27, + task28, + task29, + task30, + task31, + task32, + task33, + task35, + }, wantErr: false, }, { diff --git a/pkg/models/task_search.go b/pkg/models/task_search.go index f936a86cf..aa2fae24c 100644 --- a/pkg/models/task_search.go +++ b/pkg/models/task_search.go @@ -76,17 +76,21 @@ func getOrderByDBStatement(opts *taskSearchOptions) (orderby string, err error) return } -//nolint:gocyclo -func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) { +func convertFiltersToDBFilterCond(rawFilters []*taskFilter, includeNulls bool) (filterCond builder.Cond, err error) { - orderby, err := getOrderByDBStatement(opts) - if err != nil { - return nil, 0, err - } - - var filters = make([]builder.Cond, 0, len(opts.filters)) + var dbFilters = make([]builder.Cond, 0, len(rawFilters)) // To still find tasks with nil values, we exclude 0s when comparing with >/< values. - for _, f := range opts.filters { + for _, f := range rawFilters { + + if nested, is := f.value.([]*taskFilter); is { + nestedDBFilters, err := convertFiltersToDBFilterCond(nested, includeNulls) + if err != nil { + return nil, err + } + dbFilters = append(dbFilters, nestedDBFilters) + continue + } + if f.field == "reminders" { filter, err := getFilterCond(&taskFilter{ // recreating the struct here to avoid modifying it when reusing the opts struct @@ -94,17 +98,17 @@ func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCo value: f.value, comparator: f.comparator, isNumeric: f.isNumeric, - }, opts.filterIncludeNulls) + }, includeNulls) if err != nil { - return nil, totalCount, err + return nil, err } - filters = append(filters, getFilterCondForSeparateTable("task_reminders", filter)) + dbFilters = append(dbFilters, getFilterCondForSeparateTable("task_reminders", filter)) continue } if f.field == "assignees" { if f.comparator == taskFilterComparatorLike { - return nil, totalCount, err + return } filter, err := getFilterCond(&taskFilter{ // recreating the struct here to avoid modifying it when reusing the opts struct @@ -112,9 +116,9 @@ func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCo value: f.value, comparator: f.comparator, isNumeric: f.isNumeric, - }, opts.filterIncludeNulls) + }, includeNulls) if err != nil { - return nil, totalCount, err + return nil, err } assigneeFilter := builder.In("user_id", @@ -122,7 +126,7 @@ func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCo From("users"). Where(filter), ) - filters = append(filters, getFilterCondForSeparateTable("task_assignees", assigneeFilter)) + dbFilters = append(dbFilters, getFilterCondForSeparateTable("task_assignees", assigneeFilter)) continue } @@ -133,12 +137,12 @@ func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCo value: f.value, comparator: f.comparator, isNumeric: f.isNumeric, - }, opts.filterIncludeNulls) + }, includeNulls) if err != nil { - return nil, totalCount, err + return nil, err } - filters = append(filters, getFilterCondForSeparateTable("label_tasks", filter)) + dbFilters = append(dbFilters, getFilterCondForSeparateTable("label_tasks", filter)) continue } @@ -149,9 +153,9 @@ func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCo value: f.value, comparator: f.comparator, isNumeric: f.isNumeric, - }, opts.filterIncludeNulls) + }, includeNulls) if err != nil { - return nil, totalCount, err + return nil, err } cond := builder.In( @@ -161,15 +165,48 @@ func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCo From("projects"). Where(filter), ) - filters = append(filters, cond) + dbFilters = append(dbFilters, cond) continue } - filter, err := getFilterCond(f, opts.filterIncludeNulls) + filter, err := getFilterCond(f, includeNulls) if err != nil { - return nil, totalCount, err + return nil, err } - filters = append(filters, filter) + dbFilters = append(dbFilters, filter) + } + + if len(dbFilters) > 0 { + if len(dbFilters) == 1 { + filterCond = dbFilters[0] + } else { + for i, f := range dbFilters { + if len(dbFilters) > i+1 { + switch rawFilters[i+1].join { + case filterConcatOr: + filterCond = builder.Or(filterCond, f, dbFilters[i+1]) + case filterConcatAnd: + filterCond = builder.And(filterCond, f, dbFilters[i+1]) + } + } + } + } + } + + return filterCond, nil +} + +//nolint:gocyclo +func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) { + + orderby, err := getOrderByDBStatement(opts) + if err != nil { + return nil, 0, err + } + + filterCond, err := convertFiltersToDBFilterCond(opts.filters, opts.filterIncludeNulls) + if err != nil { + return nil, 0, err } // Then return all tasks for that projects @@ -208,24 +245,6 @@ func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCo favoritesCond = builder.In("id", favCond) } - var filterCond builder.Cond - if len(filters) > 0 { - if len(filters) == 1 { - filterCond = filters[0] - } else { - for i, f := range filters { - if len(filters) > i+1 { - switch opts.filters[i+1].join { - case filterConcatOr: - filterCond = builder.Or(filterCond, f, filters[i+1]) - case filterConcatAnd: - filterCond = builder.And(filterCond, f, filters[i+1]) - } - } - } - } - } - limit, start := getLimitFromPageIndex(opts.page, opts.perPage) cond := builder.And(builder.Or(projectIDCond, favoritesCond), where, filterCond) From 3ea81db836a773e6508cbe6903bef2d111b2f35f Mon Sep 17 00:00:00 2001 From: kolaente Date: Tue, 21 Nov 2023 19:38:06 +0100 Subject: [PATCH 10/62] feat(filter): migrate existing saved filters --- pkg/migration/20231121191822.go | 106 ++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 pkg/migration/20231121191822.go diff --git a/pkg/migration/20231121191822.go b/pkg/migration/20231121191822.go new file mode 100644 index 000000000..a0e521eea --- /dev/null +++ b/pkg/migration/20231121191822.go @@ -0,0 +1,106 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-present Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public Licensee as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public Licensee for more details. +// +// You should have received a copy of the GNU Affero General Public Licensee +// along with this program. If not, see . + +package migration + +import ( + "src.techknowlogick.com/xormigrate" + "strings" + "xorm.io/xorm" +) + +type taskCollectionFilter20231121191822 struct { + SortBy []string `query:"sort_by" json:"sort_by"` + OrderBy []string `query:"order_by" json:"order_by"` + + FilterBy []string `query:"filter_by" json:"filter_by,omitempty"` + FilterValue []string `query:"filter_value" json:"filter_value,omitempty"` + FilterComparator []string `query:"filter_comparator" json:"filter_comparator,omitempty"` + FilterConcat string `query:"filter_concat" json:"filter_concat,omitempty"` + + Filter string `query:"filter" json:"filter"` + FilterIncludeNulls bool `query:"filter_include_nulls" json:"filter_include_nulls"` +} + +type savedFilter20231121191822 struct { + ID int64 `xorm:"autoincr not null unique pk" json:"id" param:"filter"` + Filters *taskCollectionFilter20231121191822 `xorm:"JSON not null" json:"filters" valid:"required"` +} + +func (savedFilter20231121191822) TableName() string { + return "saved_filters" +} + +func init() { + migrations = append(migrations, &xormigrate.Migration{ + ID: "20231121191822", + Description: "Migrate saved filter structure", + Migrate: func(tx *xorm.Engine) (err error) { + allFilters := []*savedFilter20231121191822{} + err = tx.Find(&allFilters) + if err != nil { + return + } + + for _, filter := range allFilters { + var filterStrings []string + for i, f := range filter.Filters.FilterBy { + var comparator string + switch filter.Filters.FilterComparator[i] { + case "equals": + comparator = "=" + case "greater": + comparator = ">" + case "greater_equals": + comparator = ">=" + case "less": + comparator = "<" + case "less_equals": + comparator = "<=" + case "not_equals": + comparator = "!=" + case "like": + comparator = "~" + case "in": + comparator = "?=" + } + filterStrings = append(filterStrings, f+" "+comparator+" "+filter.Filters.FilterValue[i]) + } + + filter.Filters.FilterConcat = " || " + if filter.Filters.FilterConcat == "and" { + filter.Filters.FilterConcat = " && " + } + filter.Filters.Filter = strings.Join(filterStrings, filter.Filters.FilterConcat) + + filter.Filters.FilterBy = nil + filter.Filters.FilterComparator = nil + filter.Filters.FilterValue = nil + filter.Filters.FilterConcat = "" + + _, err = tx.Where("id = ?", filter.ID).Update(filter) + if err != nil { + return + } + } + + return nil + }, + Rollback: func(tx *xorm.Engine) error { + return nil + }, + }) +} From 9d3fb6f81d514541a9eaa8bbb58fcd66fde49f08 Mon Sep 17 00:00:00 2001 From: kolaente Date: Tue, 21 Nov 2023 19:40:13 +0100 Subject: [PATCH 11/62] chore(filter): cleanup --- pkg/models/kanban_test.go | 4 +--- pkg/models/task_collection.go | 12 ------------ pkg/models/task_collection_filter.go | 18 ------------------ pkg/models/task_collection_test.go | 9 +-------- 4 files changed, 2 insertions(+), 41 deletions(-) diff --git a/pkg/models/kanban_test.go b/pkg/models/kanban_test.go index 8ff69a9df..5993ed9ab 100644 --- a/pkg/models/kanban_test.go +++ b/pkg/models/kanban_test.go @@ -81,9 +81,7 @@ func TestBucket_ReadAll(t *testing.T) { b := &Bucket{ ProjectID: 1, TaskCollection: TaskCollection{ - FilterBy: []string{"title"}, - FilterComparator: []string{"like"}, - FilterValue: []string{"done"}, + Filter: "title ~ 'done'", }, } bucketsInterface, _, _, err := b.ReadAll(s, testuser, "", -1, 0) diff --git a/pkg/models/task_collection.go b/pkg/models/task_collection.go index 99fc58431..f8fd6cbac 100644 --- a/pkg/models/task_collection.go +++ b/pkg/models/task_collection.go @@ -33,18 +33,6 @@ type TaskCollection struct { OrderBy []string `query:"order_by" json:"order_by"` OrderByArr []string `query:"order_by[]" json:"-"` - // Deprecated: The field name of the field to filter by - FilterBy []string `query:"filter_by" json:"filter_by"` - FilterByArr []string `query:"filter_by[]" json:"-"` - // Deprecated: The value of the field name to filter by - FilterValue []string `query:"filter_value" json:"filter_value"` - FilterValueArr []string `query:"filter_value[]" json:"-"` - // Deprecated: The comparator for field and value - FilterComparator []string `query:"filter_comparator" json:"filter_comparator"` - FilterComparatorArr []string `query:"filter_comparator[]" json:"-"` - // Deprecated: The way all filter conditions are concatenated together, can be either "and" or "or"., - FilterConcat string `query:"filter_concat" json:"filter_concat"` - Filter string `query:"filter" json:"filter"` // If set to true, the result will also include null values diff --git a/pkg/models/task_collection_filter.go b/pkg/models/task_collection_filter.go index a5017edd9..32e12acf1 100644 --- a/pkg/models/task_collection_filter.go +++ b/pkg/models/task_collection_filter.go @@ -150,24 +150,6 @@ func getTaskFiltersByCollections(c *TaskCollection) (filters []*taskFilter, err return } - if len(c.FilterByArr) > 0 { - c.FilterBy = append(c.FilterBy, c.FilterByArr...) - } - - if len(c.FilterValueArr) > 0 { - c.FilterValue = append(c.FilterValue, c.FilterValueArr...) - } - - if len(c.FilterComparatorArr) > 0 { - c.FilterComparator = append(c.FilterComparator, c.FilterComparatorArr...) - } - - //if c.FilterConcat != "" && c.FilterConcat != filterConcatAnd && c.FilterConcat != filterConcatOr { - // return nil, ErrInvalidTaskFilterConcatinator{ - // Concatinator: taskFilterConcatinator(c.FilterConcat), - // } - //} - parsedFilter, err := fexpr.Parse(c.Filter) if err != nil { return nil, err diff --git a/pkg/models/task_collection_test.go b/pkg/models/task_collection_test.go index 494135fc8..6185888c8 100644 --- a/pkg/models/task_collection_test.go +++ b/pkg/models/task_collection_test.go @@ -677,12 +677,8 @@ func TestTaskCollection_ReadAll(t *testing.T) { SortBy []string // Is a string, since this is the place where a query string comes from the user OrderBy []string - FilterBy []string - FilterValue []string - FilterComparator []string FilterIncludeNulls bool - - Filter string + Filter string CRUDable web.CRUDable Rights web.Rights @@ -1392,9 +1388,6 @@ func TestTaskCollection_ReadAll(t *testing.T) { SortBy: tt.fields.SortBy, OrderBy: tt.fields.OrderBy, - FilterBy: tt.fields.FilterBy, - FilterValue: tt.fields.FilterValue, - FilterComparator: tt.fields.FilterComparator, FilterIncludeNulls: tt.fields.FilterIncludeNulls, Filter: tt.fields.Filter, From c6b682507a0198f21a2d741d2b151e54b1c591c8 Mon Sep 17 00:00:00 2001 From: kolaente Date: Tue, 21 Nov 2023 19:55:29 +0100 Subject: [PATCH 12/62] feat(filter): add better error message when passing an invalid filter expression --- docs/content/doc/usage/errors.md | 47 ++++++++++++++-------------- pkg/models/error.go | 30 +++++++++++++++++- pkg/models/task_collection_filter.go | 5 ++- 3 files changed, 57 insertions(+), 25 deletions(-) diff --git a/docs/content/doc/usage/errors.md b/docs/content/doc/usage/errors.md index 1ff2ea801..fc364ee7e 100644 --- a/docs/content/doc/usage/errors.md +++ b/docs/content/doc/usage/errors.md @@ -73,29 +73,30 @@ This document describes the different errors Vikunja can return. | ErrorCode | HTTP Status Code | Description | |-----------|------------------|----------------------------------------------------------------------------| -| 4001 | 400 | The project task text cannot be empty. | -| 4002 | 404 | The project task does not exist. | -| 4003 | 403 | All bulk editing tasks must belong to the same project. | -| 4004 | 403 | Need at least one task when bulk editing tasks. | -| 4005 | 403 | The user does not have the right to see the task. | -| 4006 | 403 | The user tried to set a parent task as the task itself. | -| 4007 | 400 | The user tried to create a task relation with an invalid kind of relation. | -| 4008 | 409 | The user tried to create a task relation which already exists. | -| 4009 | 404 | The task relation does not exist. | -| 4010 | 400 | Cannot relate a task with itself. | -| 4011 | 404 | The task attachment does not exist. | -| 4012 | 400 | The task attachment is too large. | -| 4013 | 400 | The task sort param is invalid. | -| 4014 | 400 | The task sort order is invalid. | -| 4015 | 404 | The task comment does not exist. | -| 4016 | 400 | Invalid task field. | -| 4017 | 400 | Invalid task filter comparator. | -| 4018 | 400 | Invalid task filter concatinator. | -| 4019 | 400 | Invalid task filter value. | -| 4020 | 400 | The provided attachment does not belong to that task. | -| 4021 | 400 | This user is already assigned to that task. | -| 4022 | 400 | The task has a relative reminder which does not specify relative to what. | -| 4023 | 409 | Tried to create a task relation which would create a cycle. | +| 4001 | 400 | The project task text cannot be empty. | +| 4002 | 404 | The project task does not exist. | +| 4003 | 403 | All bulk editing tasks must belong to the same project. | +| 4004 | 403 | Need at least one task when bulk editing tasks. | +| 4005 | 403 | The user does not have the right to see the task. | +| 4006 | 403 | The user tried to set a parent task as the task itself. | +| 4007 | 400 | The user tried to create a task relation with an invalid kind of relation. | +| 4008 | 409 | The user tried to create a task relation which already exists. | +| 4009 | 404 | The task relation does not exist. | +| 4010 | 400 | Cannot relate a task with itself. | +| 4011 | 404 | The task attachment does not exist. | +| 4012 | 400 | The task attachment is too large. | +| 4013 | 400 | The task sort param is invalid. | +| 4014 | 400 | The task sort order is invalid. | +| 4015 | 404 | The task comment does not exist. | +| 4016 | 400 | Invalid task field. | +| 4017 | 400 | Invalid task filter comparator. | +| 4018 | 400 | Invalid task filter concatinator. | +| 4019 | 400 | Invalid task filter value. | +| 4020 | 400 | The provided attachment does not belong to that task. | +| 4021 | 400 | This user is already assigned to that task. | +| 4022 | 400 | The task has a relative reminder which does not specify relative to what. | +| 4023 | 409 | Tried to create a task relation which would create a cycle. | +| 4024 | 400 | The provided filter expression is invalid. | ## Team diff --git a/pkg/models/error.go b/pkg/models/error.go index a3b78c445..84220540f 100644 --- a/pkg/models/error.go +++ b/pkg/models/error.go @@ -1021,7 +1021,7 @@ func (err ErrTaskRelationCycle) Error() string { } // ErrCodeTaskRelationCycle holds the unique world-error code of this error -const ErrCodeTaskRelationCycle = 4022 +const ErrCodeTaskRelationCycle = 4023 // HTTPError holds the http error description func (err ErrTaskRelationCycle) HTTPError() web.HTTPError { @@ -1032,6 +1032,34 @@ func (err ErrTaskRelationCycle) HTTPError() web.HTTPError { } } +// ErrInvalidFilterExpression represents an error where the task filter expression was invalid +type ErrInvalidFilterExpression struct { + Expression string + ExpressionError error +} + +// IsErrInvalidFilterExpression checks if an error is ErrInvalidFilterExpression. +func IsErrInvalidFilterExpression(err error) bool { + _, ok := err.(ErrInvalidFilterExpression) + return ok +} + +func (err ErrInvalidFilterExpression) Error() string { + return fmt.Sprintf("Task filter expression is invalid [ExpressionError: %v]", err.ExpressionError) +} + +// ErrCodeInvalidFilterExpression holds the unique world-error code of this error +const ErrCodeInvalidFilterExpression = 4024 + +// HTTPError holds the http error description +func (err ErrInvalidFilterExpression) HTTPError() web.HTTPError { + return web.HTTPError{ + HTTPCode: http.StatusBadRequest, + Code: ErrCodeInvalidFilterExpression, + Message: fmt.Sprintf("The filter expression '%s' is invalid: %v", err.Expression, err.ExpressionError), + } +} + // ============ // Team errors // ============ diff --git a/pkg/models/task_collection_filter.go b/pkg/models/task_collection_filter.go index 32e12acf1..21e3f52e2 100644 --- a/pkg/models/task_collection_filter.go +++ b/pkg/models/task_collection_filter.go @@ -152,7 +152,10 @@ func getTaskFiltersByCollections(c *TaskCollection) (filters []*taskFilter, err parsedFilter, err := fexpr.Parse(c.Filter) if err != nil { - return nil, err + return nil, &ErrInvalidFilterExpression{ + Expression: c.Filter, + ExpressionError: err, + } } filters = make([]*taskFilter, 0, len(parsedFilter)) From ef1cc9720c27e86d5163890f21a5f9107fc6f299 Mon Sep 17 00:00:00 2001 From: kolaente Date: Wed, 22 Nov 2023 09:24:13 +0100 Subject: [PATCH 13/62] feat(filter): add in keyword --- pkg/models/task_collection_filter.go | 2 ++ pkg/models/task_collection_test.go | 14 +++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/pkg/models/task_collection_filter.go b/pkg/models/task_collection_filter.go index 21e3f52e2..7069a70b1 100644 --- a/pkg/models/task_collection_filter.go +++ b/pkg/models/task_collection_filter.go @@ -150,6 +150,8 @@ func getTaskFiltersByCollections(c *TaskCollection) (filters []*taskFilter, err return } + c.Filter = strings.ReplaceAll(c.Filter, " in ", " ?= ") + parsedFilter, err := fexpr.Parse(c.Filter) if err != nil { return nil, &ErrInvalidFilterExpression{ diff --git a/pkg/models/task_collection_test.go b/pkg/models/task_collection_test.go index 6185888c8..9b1c95134 100644 --- a/pkg/models/task_collection_test.go +++ b/pkg/models/task_collection_test.go @@ -1032,10 +1032,22 @@ func TestTaskCollection_ReadAll(t *testing.T) { }, wantErr: false, }, + { + name: "filter in keyword", + fields: fields{ + Filter: "id in '1,2,34'", // user does not have permission to access task 34 + }, + args: defaultArgs, + want: []*Task{ + task1, + task2, + }, + wantErr: false, + }, { name: "filter in", fields: fields{ - Filter: "id ?= '1,2,34'", + Filter: "id ?= '1,2,34'", // user does not have permission to access task 34 }, args: defaultArgs, want: []*Task{ From eebfee73d3faec6745bf20ec4d2187e9880f7d19 Mon Sep 17 00:00:00 2001 From: kolaente Date: Wed, 22 Nov 2023 10:33:03 +0100 Subject: [PATCH 14/62] fix(filter): correctly filter for buckets --- pkg/models/error.go | 2 +- pkg/models/kanban.go | 37 ++++++++++++++++++---------- pkg/models/kanban_test.go | 24 ++++++++++++++++++ pkg/models/task_collection.go | 2 +- pkg/models/task_collection_filter.go | 14 +++++------ 5 files changed, 57 insertions(+), 22 deletions(-) diff --git a/pkg/models/error.go b/pkg/models/error.go index 84220540f..431d9e7d5 100644 --- a/pkg/models/error.go +++ b/pkg/models/error.go @@ -1045,7 +1045,7 @@ func IsErrInvalidFilterExpression(err error) bool { } func (err ErrInvalidFilterExpression) Error() string { - return fmt.Sprintf("Task filter expression is invalid [ExpressionError: %v]", err.ExpressionError) + return fmt.Sprintf("Task filter expression '%s' is invalid [ExpressionError: %v]", err.Expression, err.ExpressionError) } // ErrCodeInvalidFilterExpression holds the unique world-error code of this error diff --git a/pkg/models/kanban.go b/pkg/models/kanban.go index 3da2ebe20..3cf007a82 100644 --- a/pkg/models/kanban.go +++ b/pkg/models/kanban.go @@ -17,6 +17,8 @@ package models import ( + "strconv" + "strings" "time" "code.vikunja.io/api/pkg/log" @@ -175,26 +177,35 @@ func (b *Bucket) ReadAll(s *xorm.Session, auth web.Auth, search string, page int opts.search = search opts.filterConcat = filterConcatAnd - var bucketFilterIndex int - for i, filter := range opts.filters { + for _, filter := range opts.filters { if filter.field == taskPropertyBucketID { - bucketFilterIndex = i + + // Limiting the map to the one filter we're looking for is the easiest way to ensure we only + // get tasks in this bucket + bucketID := filter.value.(int64) + bucket := bucketMap[bucketID] + + bucketMap = make(map[int64]*Bucket, 1) + bucketMap[bucketID] = bucket break } } - if bucketFilterIndex == 0 { - opts.filters = append(opts.filters, &taskFilter{ - field: taskPropertyBucketID, - value: 0, - comparator: taskFilterComparatorEquals, - }) - bucketFilterIndex = len(opts.filters) - 1 - } - + originalFilter := opts.filter for id, bucket := range bucketMap { - opts.filters[bucketFilterIndex].value = id + if !strings.Contains(originalFilter, "bucket_id") { + var filterString string + if originalFilter == "" { + filterString = "bucket_id = " + strconv.FormatInt(id, 10) + } else { + filterString = "(" + originalFilter + ") && bucket_id = " + strconv.FormatInt(id, 10) + } + opts.filters, err = getTaskFiltersFromFilterString(filterString) + if err != nil { + return + } + } ts, _, total, err := getRawTasksForProjects(s, []*Project{{ID: bucket.ProjectID}}, auth, opts) if err != nil { diff --git a/pkg/models/kanban_test.go b/pkg/models/kanban_test.go index 5993ed9ab..8a1f59a87 100644 --- a/pkg/models/kanban_test.go +++ b/pkg/models/kanban_test.go @@ -92,6 +92,30 @@ func TestBucket_ReadAll(t *testing.T) { assert.Equal(t, int64(2), buckets[0].Tasks[0].ID) assert.Equal(t, int64(33), buckets[0].Tasks[1].ID) }) + t.Run("filtered by bucket", func(t *testing.T) { + db.LoadAndAssertFixtures(t) + s := db.NewSession() + defer s.Close() + + testuser := &user.User{ID: 1} + b := &Bucket{ + ProjectID: 1, + TaskCollection: TaskCollection{ + Filter: "title ~ 'task' && bucket_id = 2", + }, + } + bucketsInterface, _, _, err := b.ReadAll(s, testuser, "", -1, 0) + assert.NoError(t, err) + + buckets := bucketsInterface.([]*Bucket) + assert.Len(t, buckets, 3) + assert.Len(t, buckets[0].Tasks, 0) + assert.Len(t, buckets[1].Tasks, 3) + assert.Len(t, buckets[2].Tasks, 0) + assert.Equal(t, int64(3), buckets[1].Tasks[0].ID) + assert.Equal(t, int64(4), buckets[1].Tasks[1].ID) + assert.Equal(t, int64(5), buckets[1].Tasks[2].ID) + }) t.Run("accessed by link share", func(t *testing.T) { db.LoadAndAssertFixtures(t) s := db.NewSession() diff --git a/pkg/models/task_collection.go b/pkg/models/task_collection.go index f8fd6cbac..ab2f65321 100644 --- a/pkg/models/task_collection.go +++ b/pkg/models/task_collection.go @@ -105,7 +105,7 @@ func getTaskFilterOptsFromCollection(tf *TaskCollection) (opts *taskSearchOption filter: tf.Filter, } - opts.filters, err = getTaskFiltersByCollections(tf) + opts.filters, err = getTaskFiltersFromFilterString(tf.Filter) return opts, err } diff --git a/pkg/models/task_collection_filter.go b/pkg/models/task_collection_filter.go index 7069a70b1..3162106c4 100644 --- a/pkg/models/task_collection_filter.go +++ b/pkg/models/task_collection_filter.go @@ -144,29 +144,29 @@ func parseFilterFromExpression(f fexpr.ExprGroup) (filter *taskFilter, err error return filter, nil } -func getTaskFiltersByCollections(c *TaskCollection) (filters []*taskFilter, err error) { +func getTaskFiltersFromFilterString(filter string) (filters []*taskFilter, err error) { - if c.Filter == "" { + if filter == "" { return } - c.Filter = strings.ReplaceAll(c.Filter, " in ", " ?= ") + filter = strings.ReplaceAll(filter, " in ", " ?= ") - parsedFilter, err := fexpr.Parse(c.Filter) + parsedFilter, err := fexpr.Parse(filter) if err != nil { return nil, &ErrInvalidFilterExpression{ - Expression: c.Filter, + Expression: filter, ExpressionError: err, } } filters = make([]*taskFilter, 0, len(parsedFilter)) for _, f := range parsedFilter { - filter, err := parseFilterFromExpression(f) + parsedFilter, err := parseFilterFromExpression(f) if err != nil { return nil, err } - filters = append(filters, filter) + filters = append(filters, parsedFilter) } return From 65e1357705e0470899d9c74705326472f4481831 Mon Sep 17 00:00:00 2001 From: kolaente Date: Wed, 22 Nov 2023 10:33:27 +0100 Subject: [PATCH 15/62] fix(tests): make filter tests work again --- pkg/integrations/task_collection_test.go | 36 ++++++------------------ pkg/models/saved_filters_test.go | 2 +- 2 files changed, 10 insertions(+), 28 deletions(-) diff --git a/pkg/integrations/task_collection_test.go b/pkg/integrations/task_collection_test.go index f727489e0..40f133eb6 100644 --- a/pkg/integrations/task_collection_test.go +++ b/pkg/integrations/task_collection_test.go @@ -184,9 +184,7 @@ func TestTaskCollection(t *testing.T) { t.Run("start and end date", func(t *testing.T) { rec, err := testHandler.testReadAllWithUser( url.Values{ - "filter_by": []string{"start_date", "end_date", "due_date"}, - "filter_value": []string{"2018-12-11T03:46:40+00:00", "2018-12-13T11:20:01+00:00", "2018-11-29T14:00:00+00:00"}, - "filter_comparator": []string{"greater", "less", "greater"}, + "filter": []string{"start_date > '2018-12-11T03:46:40+00:00' || end_date < '2018-12-13T11:20:01+00:00' || due_date > '2018-11-29T14:00:00+00:00'"}, }, urlParams, ) @@ -209,9 +207,7 @@ func TestTaskCollection(t *testing.T) { t.Run("start date only", func(t *testing.T) { rec, err := testHandler.testReadAllWithUser( url.Values{ - "filter_by": []string{"start_date"}, - "filter_value": []string{"2018-10-20T01:46:40+00:00"}, - "filter_comparator": []string{"greater"}, + "filter": []string{"start_date > '2018-10-20T01:46:40+00:00'"}, }, urlParams, ) @@ -234,9 +230,7 @@ func TestTaskCollection(t *testing.T) { t.Run("end date only", func(t *testing.T) { rec, err := testHandler.testReadAllWithUser( url.Values{ - "filter_by": []string{"end_date"}, - "filter_value": []string{"2018-12-13T11:20:01+00:00"}, - "filter_comparator": []string{"greater"}, + "filter": []string{"end_date > '2018-12-13T11:20:01+00:00'"}, }, urlParams, ) @@ -249,9 +243,7 @@ func TestTaskCollection(t *testing.T) { t.Run("unix timestamps", func(t *testing.T) { rec, err := testHandler.testReadAllWithUser( url.Values{ - "filter_by": []string{"start_date", "end_date", "due_date"}, - "filter_value": []string{"1544500000", "1513164001", "1543500000"}, - "filter_comparator": []string{"greater", "less", "greater"}, + "filter": []string{"start_date > 1544500000 || end_date < 1513164001 || due_date > 1543500000"}, }, urlParams, ) @@ -275,9 +267,7 @@ func TestTaskCollection(t *testing.T) { t.Run("invalid date", func(t *testing.T) { _, err := testHandler.testReadAllWithUser( url.Values{ - "filter_by": []string{"due_date"}, - "filter_value": []string{"invalid"}, - "filter_comparator": []string{"greater"}, + "filter": []string{"due_date > invalid"}, }, nil, ) @@ -411,9 +401,7 @@ func TestTaskCollection(t *testing.T) { t.Run("start and end date", func(t *testing.T) { rec, err := testHandler.testReadAllWithUser( url.Values{ - "filter_by": []string{"start_date", "end_date", "due_date"}, - "filter_value": []string{"2018-12-11T03:46:40+00:00", "2018-12-13T11:20:01+00:00", "2018-11-29T14:00:00+00:00"}, - "filter_comparator": []string{"greater", "less", "greater"}, + "filter": []string{"start_date > '2018-12-11T03:46:40+00:00' || end_date < '2018-12-13T11:20:01+00:00' || due_date > '2018-11-29T14:00:00+00:00'"}, }, nil, ) @@ -436,9 +424,7 @@ func TestTaskCollection(t *testing.T) { t.Run("start date only", func(t *testing.T) { rec, err := testHandler.testReadAllWithUser( url.Values{ - "filter_by": []string{"start_date"}, - "filter_value": []string{"2018-10-20T01:46:40+00:00"}, - "filter_comparator": []string{"greater"}, + "filter": []string{"start_date > '2018-10-20T01:46:40+00:00'"}, }, nil, ) @@ -461,9 +447,7 @@ func TestTaskCollection(t *testing.T) { t.Run("end date only", func(t *testing.T) { rec, err := testHandler.testReadAllWithUser( url.Values{ - "filter_by": []string{"end_date"}, - "filter_value": []string{"2018-12-13T11:20:01+00:00"}, - "filter_comparator": []string{"greater"}, + "filter": []string{"end_date > '2018-12-13T11:20:01+00:00'"}, }, nil, ) @@ -477,9 +461,7 @@ func TestTaskCollection(t *testing.T) { t.Run("invalid date", func(t *testing.T) { _, err := testHandler.testReadAllWithUser( url.Values{ - "filter_by": []string{"due_date"}, - "filter_value": []string{"invalid"}, - "filter_comparator": []string{"greater"}, + "filter": []string{"due_date > invalid"}, }, nil, ) diff --git a/pkg/models/saved_filters_test.go b/pkg/models/saved_filters_test.go index 3943d0864..0fa2932f6 100644 --- a/pkg/models/saved_filters_test.go +++ b/pkg/models/saved_filters_test.go @@ -65,7 +65,7 @@ func TestSavedFilter_Create(t *testing.T) { vals := map[string]interface{}{ "title": "'test'", "description": "'Lorem Ipsum dolor sit amet'", - "filters": "'{\"sort_by\":null,\"order_by\":null,\"filter_by\":null,\"filter_value\":null,\"filter_comparator\":null,\"filter_concat\":\"\",\"filter_include_nulls\":false}'", + "filters": "'{\"sort_by\":null,\"order_by\":null,\"filter\":\"\",\"filter_include_nulls\":false}'", "owner_id": 1, } // Postgres can't compare json values directly, see https://dba.stackexchange.com/a/106290/210721 From 87c027aafd9fd86eec0b884cfa61085796f521b4 Mon Sep 17 00:00:00 2001 From: kolaente Date: Wed, 22 Nov 2023 10:43:07 +0100 Subject: [PATCH 16/62] chore(filters): cleanup old variables --- pkg/models/kanban.go | 5 ++--- pkg/models/task_collection.go | 2 +- pkg/models/task_collection_test.go | 2 ++ pkg/models/task_search.go | 4 ++-- pkg/models/tasks.go | 17 +++++------------ 5 files changed, 12 insertions(+), 18 deletions(-) diff --git a/pkg/models/kanban.go b/pkg/models/kanban.go index 3cf007a82..69cd39ff3 100644 --- a/pkg/models/kanban.go +++ b/pkg/models/kanban.go @@ -175,9 +175,8 @@ func (b *Bucket) ReadAll(s *xorm.Session, auth web.Auth, search string, page int opts.page = page opts.perPage = perPage opts.search = search - opts.filterConcat = filterConcatAnd - for _, filter := range opts.filters { + for _, filter := range opts.parsedFilters { if filter.field == taskPropertyBucketID { // Limiting the map to the one filter we're looking for is the easiest way to ensure we only @@ -201,7 +200,7 @@ func (b *Bucket) ReadAll(s *xorm.Session, auth web.Auth, search string, page int } else { filterString = "(" + originalFilter + ") && bucket_id = " + strconv.FormatInt(id, 10) } - opts.filters, err = getTaskFiltersFromFilterString(filterString) + opts.parsedFilters, err = getTaskFiltersFromFilterString(filterString) if err != nil { return } diff --git a/pkg/models/task_collection.go b/pkg/models/task_collection.go index ab2f65321..345a00608 100644 --- a/pkg/models/task_collection.go +++ b/pkg/models/task_collection.go @@ -105,7 +105,7 @@ func getTaskFilterOptsFromCollection(tf *TaskCollection) (opts *taskSearchOption filter: tf.Filter, } - opts.filters, err = getTaskFiltersFromFilterString(tf.Filter) + opts.parsedFilters, err = getTaskFiltersFromFilterString(tf.Filter) return opts, err } diff --git a/pkg/models/task_collection_test.go b/pkg/models/task_collection_test.go index 9b1c95134..7f97f2ef1 100644 --- a/pkg/models/task_collection_test.go +++ b/pkg/models/task_collection_test.go @@ -1387,6 +1387,8 @@ func TestTaskCollection_ReadAll(t *testing.T) { task9, }, }, + // TODO unix dates + // TODO date magic } for _, tt := range tests { diff --git a/pkg/models/task_search.go b/pkg/models/task_search.go index aa2fae24c..b66021f18 100644 --- a/pkg/models/task_search.go +++ b/pkg/models/task_search.go @@ -204,7 +204,7 @@ func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCo return nil, 0, err } - filterCond, err := convertFiltersToDBFilterCond(opts.filters, opts.filterIncludeNulls) + filterCond, err := convertFiltersToDBFilterCond(opts.parsedFilters, opts.filterIncludeNulls) if err != nil { return nil, 0, err } @@ -348,7 +348,7 @@ func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, "project_id: [" + strings.Join(projectIDStrings, ", ") + "]", } - for _, f := range opts.filters { + for _, f := range opts.parsedFilters { if f.field == "reminders" { f.field = "reminders.reminder" diff --git a/pkg/models/tasks.go b/pkg/models/tasks.go index 613e41246..c00ce0d00 100644 --- a/pkg/models/tasks.go +++ b/pkg/models/tasks.go @@ -167,13 +167,11 @@ const ( ) type taskSearchOptions struct { - search string - page int - perPage int - sortby []*sortParam - filters []*taskFilter - // deprecated: concat should live in filters directly - filterConcat taskFilterConcatinator + search string + page int + perPage int + sortby []*sortParam + parsedFilters []*taskFilter filterIncludeNulls bool filter string projectIDs []int64 @@ -267,11 +265,6 @@ func getRawTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, op return nil, 0, 0, nil } - // Set the default concatinator of filter variables to or if none was provided - if opts.filterConcat == "" { - opts.filterConcat = filterConcatOr - } - // Get all project IDs and get the tasks opts.projectIDs = []int64{} var hasFavoritesProject bool From bc6d812eb09185b0a42f99aacfd6af5cef49dcda Mon Sep 17 00:00:00 2001 From: kolaente Date: Wed, 22 Nov 2023 10:45:43 +0100 Subject: [PATCH 17/62] fix(filters): lint --- pkg/migration/20231121191822.go | 3 ++- pkg/models/task_collection_filter.go | 26 ++------------------------ 2 files changed, 4 insertions(+), 25 deletions(-) diff --git a/pkg/migration/20231121191822.go b/pkg/migration/20231121191822.go index a0e521eea..db7868ae1 100644 --- a/pkg/migration/20231121191822.go +++ b/pkg/migration/20231121191822.go @@ -17,8 +17,9 @@ package migration import ( - "src.techknowlogick.com/xormigrate" "strings" + + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" ) diff --git a/pkg/models/task_collection_filter.go b/pkg/models/task_collection_filter.go index 3162106c4..0e69fb2f9 100644 --- a/pkg/models/task_collection_filter.go +++ b/pkg/models/task_collection_filter.go @@ -18,12 +18,13 @@ package models import ( "fmt" - "github.com/ganigeorgiev/fexpr" "reflect" "strconv" "strings" "time" + "github.com/ganigeorgiev/fexpr" + "code.vikunja.io/api/pkg/config" "github.com/iancoleman/strcase" @@ -191,29 +192,6 @@ func validateTaskFieldComparator(comparator taskFilterComparator) error { } } -func getFilterComparatorFromString(comparator string) (taskFilterComparator, error) { - switch comparator { - case "equals": - return taskFilterComparatorEquals, nil - case "greater": - return taskFilterComparatorGreater, nil - case "greater_equals": - return taskFilterComparatorGreateEquals, nil - case "less": - return taskFilterComparatorLess, nil - case "less_equals": - return taskFilterComparatorLessEquals, nil - case "not_equals": - return taskFilterComparatorNotEquals, nil - case "like": - return taskFilterComparatorLike, nil - case "in": - return taskFilterComparatorIn, nil - default: - return taskFilterComparatorInvalid, ErrInvalidTaskFilterComparator{Comparator: taskFilterComparator(comparator)} - } -} - func getFilterComparatorFromOp(op fexpr.SignOp) (taskFilterComparator, error) { switch op { case fexpr.SignEq: From 28fa2c517a3b26a670535c05d452d606ae06e1ac Mon Sep 17 00:00:00 2001 From: kolaente Date: Wed, 22 Nov 2023 11:14:06 +0100 Subject: [PATCH 18/62] feat(filters): make new filter syntax work with Typesense --- go.mod | 1 + pkg/models/task_search.go | 112 +++++++++++++++++++++++++++----------- 2 files changed, 80 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index f65642e78..33d34fa71 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/disintegration/imaging v1.6.2 github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2 github.com/gabriel-vasile/mimetype v1.4.3 + github.com/ganigeorgiev/fexpr v0.4.0 github.com/getsentry/sentry-go v0.27.0 github.com/go-sql-driver/mysql v1.8.0 github.com/go-testfixtures/testfixtures/v3 v3.10.0 diff --git a/pkg/models/task_search.go b/pkg/models/task_search.go index b66021f18..66949dbfb 100644 --- a/pkg/models/task_search.go +++ b/pkg/models/task_search.go @@ -315,41 +315,23 @@ func convertFilterValues(value interface{}) string { return "" } -func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) { +// Parsing and rebuilding the filter for Typesense has the advantage that we have more control over +// what Typesense finally gets to see. +func convertParsedFilterToTypesense(rawFilters []*taskFilter) (filterBy string, err error) { - var sortbyFields []string - for i, param := range opts.sortby { - // Validate the params - if err := param.validate(); err != nil { - return nil, totalCount, err + filters := []string{} + + for _, f := range rawFilters { + + if nested, is := f.value.([]*taskFilter); is { + nestedDBFilters, err := convertParsedFilterToTypesense(nested) + if err != nil { + return "", err + } + filters = append(filters, "("+nestedDBFilters+")") + continue } - // Typesense does not allow sorting by ID, so we sort by created timestamp instead - if param.sortBy == "id" { - param.sortBy = "created" - } - - sortbyFields = append(sortbyFields, param.sortBy+"(missing_values:last):"+param.orderBy.String()) - - if i == 2 { - // Typesense supports up to 3 sorting parameters - // https://typesense.org/docs/0.25.0/api/search.html#ranking-and-sorting-parameters - break - } - } - - sortby := strings.Join(sortbyFields, ",") - - projectIDStrings := []string{} - for _, id := range opts.projectIDs { - projectIDStrings = append(projectIDStrings, strconv.FormatInt(id, 10)) - } - filterBy := []string{ - "project_id: [" + strings.Join(projectIDStrings, ", ") + "]", - } - - for _, f := range opts.parsedFilters { - if f.field == "reminders" { f.field = "reminders.reminder" } @@ -362,6 +344,10 @@ func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, f.field = "labels.id" } + if f.field == "project" { + f.field = "project_id" + } + filter := f.field switch f.comparator { @@ -393,7 +379,67 @@ func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, filter += "]" } - filterBy = append(filterBy, filter) + filters = append(filters, filter) + } + + if len(filters) > 0 { + if len(filters) == 1 { + filterBy = filters[0] + } else { + for i, f := range filters { + if len(filters) > i+1 { + switch rawFilters[i+1].join { + case filterConcatOr: + filterBy = f + " || " + filters[i+1] + case filterConcatAnd: + filterBy = f + " && " + filters[i+1] + } + } + } + } + } + + return +} + +func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) { + + var sortbyFields []string + for i, param := range opts.sortby { + // Validate the params + if err := param.validate(); err != nil { + return nil, totalCount, err + } + + // Typesense does not allow sorting by ID, so we sort by created timestamp instead + if param.sortBy == "id" { + param.sortBy = "created" + } + + sortbyFields = append(sortbyFields, param.sortBy+"(missing_values:last):"+param.orderBy.String()) + + if i == 2 { + // Typesense supports up to 3 sorting parameters + // https://typesense.org/docs/0.25.0/api/search.html#ranking-and-sorting-parameters + break + } + } + + sortby := strings.Join(sortbyFields, ",") + + projectIDStrings := []string{} + for _, id := range opts.projectIDs { + projectIDStrings = append(projectIDStrings, strconv.FormatInt(id, 10)) + } + + filter, err := convertParsedFilterToTypesense(opts.parsedFilters) + if err != nil { + return nil, 0, err + } + + filterBy := []string{ + "project_id: [" + strings.Join(projectIDStrings, ", ") + "]", + "(" + filter + ")", } //////////////// From b978d344caeb4a63cffca103d6984ab58882f3dd Mon Sep 17 00:00:00 2001 From: kolaente Date: Fri, 1 Dec 2023 17:13:49 +0100 Subject: [PATCH 19/62] feat(filter): add basic highlighting filter query component --- .../project/partials/FilterInput.vue | 130 ++++++++++++++++++ .../components/project/partials/filters.vue | 7 + frontend/src/i18n/lang/en.json | 3 + 3 files changed, 140 insertions(+) create mode 100644 frontend/src/components/project/partials/FilterInput.vue diff --git a/frontend/src/components/project/partials/FilterInput.vue b/frontend/src/components/project/partials/FilterInput.vue new file mode 100644 index 000000000..52e41acd9 --- /dev/null +++ b/frontend/src/components/project/partials/FilterInput.vue @@ -0,0 +1,130 @@ + + + + + + + diff --git a/frontend/src/components/project/partials/filters.vue b/frontend/src/components/project/partials/filters.vue index a9058fb21..29535bf20 100644 --- a/frontend/src/components/project/partials/filters.vue +++ b/frontend/src/components/project/partials/filters.vue @@ -30,6 +30,9 @@ {{ $t('filters.attributes.sortAlphabetically') }} + + +
@@ -227,6 +230,7 @@ import ProjectService from '@/services/project' // FIXME: do not use this here for now. instead create new version from DEFAULT_PARAMS import {getDefaultParams} from '@/composables/useTaskList' +import FilterInput from '@/components/project/partials/FilterInput.vue' const props = defineProps({ modelValue: { @@ -252,6 +256,9 @@ const DEFAULT_PARAMS = { s: '', } as const +// FIXME: use params +const filterQuery = ref('') + const DEFAULT_FILTERS = { done: false, dueDate: '', diff --git a/frontend/src/i18n/lang/en.json b/frontend/src/i18n/lang/en.json index fdc09d2cb..ae2a5ca59 100644 --- a/frontend/src/i18n/lang/en.json +++ b/frontend/src/i18n/lang/en.json @@ -415,6 +415,9 @@ "edit": { "title": "Edit This Saved Filter", "success": "The filter was saved successfully." + }, + "query": { + "title": "Query" } }, "migrate": { From c162a5a45721a2df4a897b48cfe4179dd4a43bfc Mon Sep 17 00:00:00 2001 From: kolaente Date: Fri, 15 Dec 2023 19:01:08 +0100 Subject: [PATCH 20/62] feat(filter): add auto resize for filter query input --- .../project/partials/FilterInput.vue | 20 +++++++++++++++---- frontend/src/components/tasks/add-task.vue | 2 +- .../src/composables/useAutoHeightTextarea.ts | 13 +++++++----- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/frontend/src/components/project/partials/FilterInput.vue b/frontend/src/components/project/partials/FilterInput.vue index 52e41acd9..d58ad687e 100644 --- a/frontend/src/components/project/partials/FilterInput.vue +++ b/frontend/src/components/project/partials/FilterInput.vue @@ -1,5 +1,6 @@ + + From 3bd639a11073fecd14275a80f23fe764f7d061a2 Mon Sep 17 00:00:00 2001 From: kolaente Date: Sat, 17 Feb 2024 16:57:55 +0100 Subject: [PATCH 24/62] chore(filters): copy datepicker --- .../components/date/datepickerWithValues.vue | 302 ++++++++++++++++++ 1 file changed, 302 insertions(+) create mode 100644 frontend/src/components/date/datepickerWithValues.vue diff --git a/frontend/src/components/date/datepickerWithValues.vue b/frontend/src/components/date/datepickerWithValues.vue new file mode 100644 index 000000000..be9e70683 --- /dev/null +++ b/frontend/src/components/date/datepickerWithValues.vue @@ -0,0 +1,302 @@ + + + + + From c22daab28cbaef9624885ea4af282cf468307605 Mon Sep 17 00:00:00 2001 From: kolaente Date: Sat, 17 Feb 2024 17:48:22 +0100 Subject: [PATCH 25/62] feat(filters): make date values in filter query editable --- .../components/date/datepickerWithRange.vue | 17 +-- .../components/date/datepickerWithValues.vue | 123 ++++++------------ frontend/src/components/misc/popup.vue | 13 +- .../partials}/FilterInput.story.vue | 12 +- .../project/partials/FilterInput.vue | 122 ++++++++++++----- 5 files changed, 156 insertions(+), 131 deletions(-) rename frontend/src/components/{input => project/partials}/FilterInput.story.vue (50%) diff --git a/frontend/src/components/date/datepickerWithRange.vue b/frontend/src/components/date/datepickerWithRange.vue index be9e70683..83a853f01 100644 --- a/frontend/src/components/date/datepickerWithRange.vue +++ b/frontend/src/components/date/datepickerWithRange.vue @@ -75,14 +75,15 @@

{{ $t('input.datemathHelp.canuse') }} - - {{ $t('input.datemathHelp.learnhow') }} -

+ + {{ $t('input.datemathHelp.learnhow') }} + + - +
@@ -111,7 +112,7 @@ import Popup from '@/components/misc/popup.vue' import {DATE_RANGES} from '@/components/date/dateRanges' import BaseButton from '@/components/base/BaseButton.vue' import DatemathHelp from '@/components/date/datemathHelp.vue' -import { getFlatpickrLanguage } from '@/helpers/flatpickrLanguage' +import {getFlatpickrLanguage} from '@/helpers/flatpickrLanguage' const props = defineProps({ modelValue: { diff --git a/frontend/src/components/date/datepickerWithValues.vue b/frontend/src/components/date/datepickerWithValues.vue index be9e70683..96c827a01 100644 --- a/frontend/src/components/date/datepickerWithValues.vue +++ b/frontend/src/components/date/datepickerWithValues.vue @@ -1,13 +1,9 @@ From 992d108bfacbb5808069d940a0aa4ff67beba7e6 Mon Sep 17 00:00:00 2001 From: kolaente Date: Sat, 17 Feb 2024 17:57:50 +0100 Subject: [PATCH 26/62] feat(filters): add date values --- frontend/src/components/date/dateRanges.ts | 25 +++++++++++++++++++ .../components/date/datepickerWithValues.vue | 12 ++++----- frontend/src/i18n/lang/en.json | 25 +++++++++++++++++++ 3 files changed, 56 insertions(+), 6 deletions(-) diff --git a/frontend/src/components/date/dateRanges.ts b/frontend/src/components/date/dateRanges.ts index 648f001b4..556c21c1f 100644 --- a/frontend/src/components/date/dateRanges.ts +++ b/frontend/src/components/date/dateRanges.ts @@ -19,3 +19,28 @@ export const DATE_RANGES = { 'thisYear': ['now/y', 'now/y+1y'], 'restOfThisYear': ['now', 'now/y+1y'], } + +export const DATE_VALUES = { + 'now': 'now', + 'startOfToday': 'now/d', + 'endOfToday': 'now/d+1d', + + 'beginningOflastWeek': 'now/w-1w', + 'endOfLastWeek': 'now/w-2w', + 'beginningOfThisWeek': 'now/w', + 'endOfThisWeek': 'now/w+1w', + 'startOfNextWeek': 'now/w+1w', + 'endOfNextWeek': 'now/w+2w', + 'in7Days': 'now+7d', + + 'beginningOfLastMonth': 'now/M-1M', + 'endOfLastMonth': 'now/M-2M', + 'startOfThisMonth': 'now/M', + 'endOfThisMonth': 'now/M+1M', + 'startOfNextMonth': 'now/M+1M', + 'endOfNextMonth': 'now/M+2M', + 'in30Days': 'now+30d', + + 'startOfThisYear': 'now/y', + 'endOfThisYear': 'now/y+1y', +} diff --git a/frontend/src/components/date/datepickerWithValues.vue b/frontend/src/components/date/datepickerWithValues.vue index 96c827a01..dd9259103 100644 --- a/frontend/src/components/date/datepickerWithValues.vue +++ b/frontend/src/components/date/datepickerWithValues.vue @@ -17,12 +17,12 @@ {{ $t('misc.custom') }} - {{ $t(`input.datepickerRange.ranges.${text}`) }} + {{ $t(`input.datepickerRange.values.${text}`) }}
@@ -86,7 +86,7 @@ import 'flatpickr/dist/flatpickr.css' import {parseDateOrString} from '@/helpers/time/parseDateOrString' import Popup from '@/components/misc/popup.vue' -import {DATE_RANGES} from '@/components/date/dateRanges' +import {DATE_VALUES} from '@/components/date/dateRanges' import BaseButton from '@/components/base/BaseButton.vue' import DatemathHelp from '@/components/date/datemathHelp.vue' import {getFlatpickrLanguage} from '@/helpers/flatpickrLanguage' @@ -162,7 +162,7 @@ function setDate(range: string | null) { } const customRangeActive = computed(() => { - return !Object.values(DATE_RANGES).some(d => date.value === d) + return !Object.values(DATE_VALUES).some(d => date.value === d) }) diff --git a/frontend/src/i18n/lang/en.json b/frontend/src/i18n/lang/en.json index ae2a5ca59..a514b969c 100644 --- a/frontend/src/i18n/lang/en.json +++ b/frontend/src/i18n/lang/en.json @@ -588,6 +588,7 @@ "to": "To", "from": "From", "fromto": "{from} to {to}", + "date": "Date", "ranges": { "today": "Today", @@ -605,6 +606,30 @@ "thisYear": "This Year", "restOfThisYear": "The Rest of This Year" + }, + "values": { + "now": "Now", + "startOfToday": "Start of today", + "endOfToday": "End of today", + + "beginningOflastWeek": "Beginning of last week", + "endOfLastWeek": "End of last week", + "beginningOfThisWeek": "Beginning of this week", + "endOfThisWeek": "End of this week", + "startOfNextWeek": "Start of next week", + "endOfNextWeek": "End of next week", + "in7Days": "In 7 days", + + "beginningOfLastMonth": "Beginning of last month", + "endOfLastMonth": "End of last month", + "startOfThisMonth": "Start of this month", + "endOfThisMonth": "End of this month", + "startOfNextMonth": "Start of next month", + "endOfNextMonth": "End of next month", + "in30Days": "In 30 days", + + "startOfThisYear": "Beginning of this year", + "endOfThisYear": "End of this year" } }, "datemathHelp": { From 388a3a68bacf24dafe808a881d5d842140be6363 Mon Sep 17 00:00:00 2001 From: kolaente Date: Sat, 17 Feb 2024 18:44:37 +0100 Subject: [PATCH 27/62] fix(filters): date filter value not populated --- frontend/src/components/project/partials/FilterInput.vue | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/components/project/partials/FilterInput.vue b/frontend/src/components/project/partials/FilterInput.vue index ed252c883..30a72dc71 100644 --- a/frontend/src/components/project/partials/FilterInput.vue +++ b/frontend/src/components/project/partials/FilterInput.vue @@ -125,6 +125,7 @@ watch( const button = event.target currentOldDatepickerValue.value = button?.innerText + currentDatepickerValue.value = button?.innerText currentDatepickerPos.value = parseInt(button?.dataset.position) datePickerPopupOpen.value = true }) From 571bcf8996a1e11e03d65969fdeec17339d9acfb Mon Sep 17 00:00:00 2001 From: kolaente Date: Sat, 17 Feb 2024 19:10:24 +0100 Subject: [PATCH 28/62] feat(filters): show user name and avatar for assignee filters --- .../project/partials/FilterInput.vue | 66 +++++++++++++++++-- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/frontend/src/components/project/partials/FilterInput.vue b/frontend/src/components/project/partials/FilterInput.vue index 30a72dc71..101abc5ba 100644 --- a/frontend/src/components/project/partials/FilterInput.vue +++ b/frontend/src/components/project/partials/FilterInput.vue @@ -2,6 +2,9 @@ import {computed, nextTick, ref, watch} from 'vue' import {useAutoHeightTextarea} from '@/composables/useAutoHeightTextarea' import DatepickerWithValues from '@/components/date/datepickerWithValues.vue' +import UserService from "@/services/user"; +import {getAvatarUrl, getDisplayName} from "@/models/user"; +import {createRandomID} from "@/helpers/randomId"; const { modelValue, @@ -23,11 +26,18 @@ watch( {immediate: true}, ) +const userService = new UserService() + const dateFields = [ 'dueDate', 'startDate', 'endDate', 'doneAt', + 'reminders', +] + +const assigneeFields = [ + 'assignees', ] const availableFilterFields = [ @@ -35,10 +45,9 @@ const availableFilterFields = [ 'priority', 'usePriority', 'percentDone', - 'reminders', - 'assignees', 'labels', ...dateFields, + ...assigneeFields, ] const filterOperators = [ @@ -81,17 +90,51 @@ function unEscapeHtml(unsafe: string): string { const highlightedFilterQuery = computed(() => { let highlighted = escapeHtml(filterQuery.value) dateFields - .map(o => escapeHtml(o)) .forEach(o => { const pattern = new RegExp(o + '\\s*(<|>|<=|>=|=|!=)\\s*([\'"]?)([^\'"\\s]+\\1?)?', 'ig'); - highlighted = highlighted.replaceAll(pattern, (match, token, start, value, position, last) => { - console.log({position, last}) + highlighted = highlighted.replaceAll(pattern, (match, token, start, value, position) => { if (typeof value === 'undefined') { value = '' } return `${o} ${token} ${value}` }) }) + assigneeFields + .forEach(f => { + const pattern = new RegExp(f + '\\s*(<|>|<=|>=|=|!=)\\s*([\'"]?)([^\'"\\s]+\\1?)?', 'ig'); + highlighted = highlighted.replaceAll(pattern, (match, token, start, value) => { + if (typeof value === 'undefined') { + value = '' + } + + const id = createRandomID(32) + + userService.getAll({}, {s: value}).then(users => { + if (users.length > 0) { + const displayName = getDisplayName(users[0]) + const nameTag = document.createElement('span') + nameTag.innerText = displayName + + const avatar = document.createElement('img') + avatar.src = getAvatarUrl(users[0], 20) + avatar.height = 20 + avatar.width = 20 + avatar.alt = displayName + + // TODO: caching + + nextTick(() => { + const assigneeValue = document.getElementById(id) + assigneeValue.innerText = '' + assigneeValue?.appendChild(avatar) + assigneeValue?.appendChild(nameTag) + }) + } + }) + + return `${f} ${token} ${value}` + }) + }) filterOperators .map(o => ` ${escapeHtml(o)} `) .forEach(o => { @@ -194,6 +237,19 @@ function updateDateInQuery(newDate: string) { padding: .125rem .25rem; display: inline-block; } + + &.filter-query__assignee_value { + padding: .125rem .25rem; + border-radius: $radius; + background-color: var(--grey-200); + color: var(--grey-700); + display: inline-flex; + align-items: center; + + > img { + margin-right: .25rem; + } + } } button.filter-query__date_value { From 35487093c6ac7372f1a37195a0233f0d1b9c5016 Mon Sep 17 00:00:00 2001 From: kolaente Date: Tue, 5 Mar 2024 09:58:36 +0100 Subject: [PATCH 29/62] chore: update lockfile --- go.sum | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/go.sum b/go.sum index eecaface0..62e4b28c5 100644 --- a/go.sum +++ b/go.sum @@ -109,12 +109,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= -github.com/ganigeorgiev/fexpr v0.3.0 h1:RwSyJBME+g/XdzlUW0paH/4VXhLHPg+rErtLeC7K8Ew= -github.com/ganigeorgiev/fexpr v0.3.0/go.mod h1:RyGiGqmeXhEQ6+mlGdnUleLHgtzzu/VGO2WtJkF5drE= -github.com/getsentry/sentry-go v0.25.0 h1:q6Eo+hS+yoJlTO3uu/azhQadsD8V+jQn2D8VvX1eOyI= -github.com/getsentry/sentry-go v0.25.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= -github.com/getsentry/sentry-go v0.26.0 h1:IX3++sF6/4B5JcevhdZfdKIHfyvMmAq/UnqcyT2H6mA= -github.com/getsentry/sentry-go v0.26.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/ganigeorgiev/fexpr v0.4.0 h1:ojitI+VMNZX/odeNL1x3RzTTE8qAIVvnSSYPNAnQFDI= +github.com/ganigeorgiev/fexpr v0.4.0/go.mod h1:RyGiGqmeXhEQ6+mlGdnUleLHgtzzu/VGO2WtJkF5drE= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= From 2daecbc2bc943a4e6c091559f7fd7c4c3601c7c2 Mon Sep 17 00:00:00 2001 From: kolaente Date: Tue, 5 Mar 2024 10:37:26 +0100 Subject: [PATCH 30/62] feat(filters): add basic autocomplete component --- .../src/components/input/Autocomplete.vue | 316 ++++++++++++++++++ 1 file changed, 316 insertions(+) create mode 100644 frontend/src/components/input/Autocomplete.vue diff --git a/frontend/src/components/input/Autocomplete.vue b/frontend/src/components/input/Autocomplete.vue new file mode 100644 index 000000000..ea8a27b60 --- /dev/null +++ b/frontend/src/components/input/Autocomplete.vue @@ -0,0 +1,316 @@ + + + + + \ No newline at end of file From 2990c01d0a53c3a26de4737e2c09b57baaf35f18 Mon Sep 17 00:00:00 2001 From: kolaente Date: Tue, 5 Mar 2024 18:34:55 +0100 Subject: [PATCH 31/62] fix(filters): make the button look less like a button to avoid spacing problems --- .../src/components/project/partials/FilterInput.vue | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/project/partials/FilterInput.vue b/frontend/src/components/project/partials/FilterInput.vue index 101abc5ba..6a14498f4 100644 --- a/frontend/src/components/project/partials/FilterInput.vue +++ b/frontend/src/components/project/partials/FilterInput.vue @@ -96,7 +96,7 @@ const highlightedFilterQuery = computed(() => { if (typeof value === 'undefined') { value = '' } - return `${o} ${token} ${value}` + return `${o} ${token} ${value}` }) }) assigneeFields @@ -234,8 +234,8 @@ function updateDateInQuery(newDate: string) { } &.filter-query__date_value_placeholder { - padding: .125rem .25rem; display: inline-block; + color: transparent; } &.filter-query__assignee_value { @@ -253,11 +253,17 @@ function updateDateInQuery(newDate: string) { } button.filter-query__date_value { - padding: .125rem .25rem; border-radius: $radius; position: absolute; margin-top: calc((0.25em - 0.125rem) * -1); height: 1.75rem; + padding: 0; + border: 0; + background: transparent; + color: var(--primary); + font-size: 1rem; + cursor: pointer; + line-height: 1.5; } } From 981f2d0e7066cc511e4babab504e860a6750430e Mon Sep 17 00:00:00 2001 From: kolaente Date: Tue, 5 Mar 2024 18:49:57 +0100 Subject: [PATCH 32/62] fix(filters): color --- frontend/src/components/project/partials/FilterInput.vue | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/src/components/project/partials/FilterInput.vue b/frontend/src/components/project/partials/FilterInput.vue index 6a14498f4..58e3cdbac 100644 --- a/frontend/src/components/project/partials/FilterInput.vue +++ b/frontend/src/components/project/partials/FilterInput.vue @@ -260,7 +260,6 @@ function updateDateInQuery(newDate: string) { padding: 0; border: 0; background: transparent; - color: var(--primary); font-size: 1rem; cursor: pointer; line-height: 1.5; From 9ed93b181daa6bc248343b22185741728dea55f6 Mon Sep 17 00:00:00 2001 From: kolaente Date: Tue, 5 Mar 2024 18:56:47 +0100 Subject: [PATCH 33/62] fix(filters): make sure spaces before and after are not removed --- frontend/src/components/project/partials/FilterInput.vue | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/project/partials/FilterInput.vue b/frontend/src/components/project/partials/FilterInput.vue index 58e3cdbac..704a85b23 100644 --- a/frontend/src/components/project/partials/FilterInput.vue +++ b/frontend/src/components/project/partials/FilterInput.vue @@ -91,12 +91,13 @@ const highlightedFilterQuery = computed(() => { let highlighted = escapeHtml(filterQuery.value) dateFields .forEach(o => { - const pattern = new RegExp(o + '\\s*(<|>|<=|>=|=|!=)\\s*([\'"]?)([^\'"\\s]+\\1?)?', 'ig'); - highlighted = highlighted.replaceAll(pattern, (match, token, start, value, position) => { + const pattern = new RegExp(o + '(\\s*)(<|>|<=|>=|=|!=)(\\s*)([\'"]?)([^\'"\\s]+\\1?)?', 'ig'); + highlighted = highlighted.replaceAll(pattern, (match, spacesBefore, token, spacesAfter, start, value, position) => { if (typeof value === 'undefined') { value = '' } - return `${o} ${token} ${value}` + + return `${o}${spacesBefore}${token}${spacesAfter}${value}` }) }) assigneeFields From 356399f8539da43cedf085a7917a40705f9d2d09 Mon Sep 17 00:00:00 2001 From: kolaente Date: Tue, 5 Mar 2024 18:57:11 +0100 Subject: [PATCH 34/62] chore: format --- .../project/partials/FilterInput.vue | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/frontend/src/components/project/partials/FilterInput.vue b/frontend/src/components/project/partials/FilterInput.vue index 704a85b23..667564db4 100644 --- a/frontend/src/components/project/partials/FilterInput.vue +++ b/frontend/src/components/project/partials/FilterInput.vue @@ -2,9 +2,9 @@ import {computed, nextTick, ref, watch} from 'vue' import {useAutoHeightTextarea} from '@/composables/useAutoHeightTextarea' import DatepickerWithValues from '@/components/date/datepickerWithValues.vue' -import UserService from "@/services/user"; -import {getAvatarUrl, getDisplayName} from "@/models/user"; -import {createRandomID} from "@/helpers/randomId"; +import UserService from '@/services/user' +import {getAvatarUrl, getDisplayName} from '@/models/user' +import {createRandomID} from '@/helpers/randomId' const { modelValue, @@ -84,46 +84,46 @@ function unEscapeHtml(unsafe: string): string { .replace(/</g, '<') .replace(/>/g, '>') .replace(/"/g, '"') - .replace(/'/g, "'") + .replace(/'/g, '\'') } const highlightedFilterQuery = computed(() => { let highlighted = escapeHtml(filterQuery.value) dateFields .forEach(o => { - const pattern = new RegExp(o + '(\\s*)(<|>|<=|>=|=|!=)(\\s*)([\'"]?)([^\'"\\s]+\\1?)?', 'ig'); + const pattern = new RegExp(o + '(\\s*)(<|>|<=|>=|=|!=)(\\s*)([\'"]?)([^\'"\\s]+\\1?)?', 'ig') highlighted = highlighted.replaceAll(pattern, (match, spacesBefore, token, spacesAfter, start, value, position) => { if (typeof value === 'undefined') { value = '' } - + return `${o}${spacesBefore}${token}${spacesAfter}${value}` }) }) assigneeFields .forEach(f => { - const pattern = new RegExp(f + '\\s*(<|>|<=|>=|=|!=)\\s*([\'"]?)([^\'"\\s]+\\1?)?', 'ig'); + const pattern = new RegExp(f + '\\s*(<|>|<=|>=|=|!=)\\s*([\'"]?)([^\'"\\s]+\\1?)?', 'ig') highlighted = highlighted.replaceAll(pattern, (match, token, start, value) => { if (typeof value === 'undefined') { value = '' } - + const id = createRandomID(32) - + userService.getAll({}, {s: value}).then(users => { if (users.length > 0) { const displayName = getDisplayName(users[0]) const nameTag = document.createElement('span') nameTag.innerText = displayName - + const avatar = document.createElement('img') avatar.src = getAvatarUrl(users[0], 20) avatar.height = 20 avatar.width = 20 avatar.alt = displayName - + // TODO: caching - + nextTick(() => { const assigneeValue = document.getElementById(id) assigneeValue.innerText = '' @@ -132,7 +132,7 @@ const highlightedFilterQuery = computed(() => { }) } }) - + return `${f} ${token} ${value}` }) }) @@ -175,12 +175,12 @@ watch( }) }) }, - {immediate: true} + {immediate: true}, ) function updateDateInQuery(newDate: string) { // Need to escape and unescape the query because the positions are based on the escaped query - let escaped = escapeHtml(filterQuery.value) + let escaped = escapeHtml(filterQuery.value) escaped = escaped .substring(0, currentDatepickerPos.value) + escaped @@ -238,7 +238,7 @@ function updateDateInQuery(newDate: string) { display: inline-block; color: transparent; } - + &.filter-query__assignee_value { padding: .125rem .25rem; border-radius: $radius; @@ -246,7 +246,7 @@ function updateDateInQuery(newDate: string) { color: var(--grey-700); display: inline-flex; align-items: center; - + > img { margin-right: .25rem; } From 7fc1f27ef5d8d95f1ec5585a755f2e6ad2b68233 Mon Sep 17 00:00:00 2001 From: kolaente Date: Wed, 6 Mar 2024 17:59:00 +0100 Subject: [PATCH 35/62] feat(filter): add autocompletion poc for labels --- .../components/input/AutocompleteDropdown.vue | 237 ++++++++++++++++++ .../project/partials/FilterInput.vue | 95 +++++-- 2 files changed, 307 insertions(+), 25 deletions(-) create mode 100644 frontend/src/components/input/AutocompleteDropdown.vue diff --git a/frontend/src/components/input/AutocompleteDropdown.vue b/frontend/src/components/input/AutocompleteDropdown.vue new file mode 100644 index 000000000..096ee09c7 --- /dev/null +++ b/frontend/src/components/input/AutocompleteDropdown.vue @@ -0,0 +1,237 @@ + + + diff --git a/frontend/src/components/project/partials/FilterInput.vue b/frontend/src/components/project/partials/FilterInput.vue index 667564db4..06b32a684 100644 --- a/frontend/src/components/project/partials/FilterInput.vue +++ b/frontend/src/components/project/partials/FilterInput.vue @@ -5,6 +5,7 @@ import DatepickerWithValues from '@/components/date/datepickerWithValues.vue' import UserService from '@/services/user' import {getAvatarUrl, getDisplayName} from '@/models/user' import {createRandomID} from '@/helpers/randomId' +import AutocompleteDropdown from '@/components/input/AutocompleteDropdown.vue' const { modelValue, @@ -40,14 +41,18 @@ const assigneeFields = [ 'assignees', ] +const labelFields = [ + 'labels', +] + const availableFilterFields = [ 'done', 'priority', 'usePriority', 'percentDone', - 'labels', ...dateFields, ...assigneeFields, + ...labelFields, ] const filterOperators = [ @@ -69,6 +74,9 @@ const filterJoinOperators = [ ')', ] +const FILTER_OPERATORS_REGEX = '(<|>|<=|>=|=|!=)' +const FILTER_JOIN_OPERATORS_REGEX = '(&&|\|\||\(|\))' + function escapeHtml(unsafe: string): string { return unsafe .replace(/&/g, '&') @@ -91,7 +99,7 @@ const highlightedFilterQuery = computed(() => { let highlighted = escapeHtml(filterQuery.value) dateFields .forEach(o => { - const pattern = new RegExp(o + '(\\s*)(<|>|<=|>=|=|!=)(\\s*)([\'"]?)([^\'"\\s]+\\1?)?', 'ig') + const pattern = new RegExp(o + '(\\s*)' + FILTER_OPERATORS_REGEX + '(\\s*)([\'"]?)([^\'"\\s]+\\1?)?', 'ig') highlighted = highlighted.replaceAll(pattern, (match, spacesBefore, token, spacesAfter, start, value, position) => { if (typeof value === 'undefined') { value = '' @@ -102,7 +110,7 @@ const highlightedFilterQuery = computed(() => { }) assigneeFields .forEach(f => { - const pattern = new RegExp(f + '\\s*(<|>|<=|>=|=|!=)\\s*([\'"]?)([^\'"\\s]+\\1?)?', 'ig') + const pattern = new RegExp(f + '\\s*' + FILTER_OPERATORS_REGEX + '\\s*([\'"]?)([^\'"\\s]+\\1?)?', 'ig') highlighted = highlighted.replaceAll(pattern, (match, token, start, value) => { if (typeof value === 'undefined') { value = '' @@ -189,33 +197,70 @@ function updateDateInQuery(newDate: string) { currentOldDatepickerValue.value = newDate filterQuery.value = unEscapeHtml(escaped) } + +function handleFieldInput(e, autocompleteOnInput) { + const cursorPosition = filterInput.value.selectionStart + const textUpToCursor = filterQuery.value.substring(0, cursorPosition) + + labelFields.forEach(l => { + const pattern = new RegExp('(' + l + '\\s*' + FILTER_OPERATORS_REGEX + '\\s*)([\'"]?)([^\'"&\|\(\)]+\\1?)?$', 'ig') + const match = pattern.exec(textUpToCursor) + + if (match !== null) { + const [matched, prefix, operator, space, keyword] = match + if (keyword) { + autocompleteResults.value = ['loool', keyword] + } + } + }) +} + +const autocompleteResults = ref([]) From 9ade917ac455d830bdb1733c77e8f9b4fd7a94d9 Mon Sep 17 00:00:00 2001 From: kolaente Date: Wed, 6 Mar 2024 18:09:17 +0100 Subject: [PATCH 36/62] feat(filter): make the autocomplete look pretty --- .../components/input/AutocompleteDropdown.vue | 79 ++++++++++++------- .../project/partials/FilterInput.vue | 7 +- 2 files changed, 56 insertions(+), 30 deletions(-) diff --git a/frontend/src/components/input/AutocompleteDropdown.vue b/frontend/src/components/input/AutocompleteDropdown.vue index 096ee09c7..927d7fa14 100644 --- a/frontend/src/components/input/AutocompleteDropdown.vue +++ b/frontend/src/components/input/AutocompleteDropdown.vue @@ -12,7 +12,6 @@ type state = 'unfocused' | 'focused' const selectedIndex = ref(-1) const state = ref('unfocused') const val = ref('') -const isResizing = ref(false) const model = defineModel() const suggestionScrollerRef = ref(null) @@ -77,38 +76,10 @@ function updateSuggestionScroll() { }) } -function updateScrollWindowSize() { - if (isResizing.value) { - return - } - - isResizing.value = true - - nextTick(() => { - isResizing.value = false - - const scroller = suggestionScrollerRef.value - const parent = containerRef.value - if (scroller) { - const rect = parent.getBoundingClientRect() - const pxTop = rect.top - const pxBottom = window.innerHeight - rect.bottom - const maxHeight = Math.max(pxTop, pxBottom, props.maxHeight) - const isReversed = pxBottom < props.maxHeight && pxTop > pxBottom - scroller.style.maxHeight = Math.min(isReversed ? pxTop : pxBottom, props.maxHeight) + 'px' - scroller.parentNode.style.transform = - isReversed ? 'translateY(-100%) translateY(-1.4rem)' - : 'translateY(.4rem)' - } - }) -} - function setState(stateName: state) { state.value = stateName if (stateName === 'unfocused') { emit('blur') - } else { - updateScrollWindowSize() } } @@ -235,3 +206,53 @@ function onUpdateField(e) {
+ + diff --git a/frontend/src/components/project/partials/FilterInput.vue b/frontend/src/components/project/partials/FilterInput.vue index 06b32a684..9fbffd514 100644 --- a/frontend/src/components/project/partials/FilterInput.vue +++ b/frontend/src/components/project/partials/FilterInput.vue @@ -240,6 +240,7 @@ const autocompleteResults = ref([]) spellcheck="false" v-model="filterQuery" class="input" + :class="{'has-autocomplete-results': autocompleteResults.length > 0}" ref="filterInput" >
([])
@@ -323,6 +324,10 @@ const autocompleteResults = ref([]) -webkit-text-fill-color: transparent; background: transparent !important; resize: none; + + &.has-autocomplete-results { + border-radius: var(--input-radius) var(--input-radius) 0 0; + } } .filter-input-highlight { From 8fa2f6686a860195517d8f0a417c82783a36ae8b Mon Sep 17 00:00:00 2001 From: kolaente Date: Wed, 6 Mar 2024 18:33:31 +0100 Subject: [PATCH 37/62] feat(filter): add actual label search when autocompleting --- .../components/input/AutocompleteDropdown.vue | 28 ---------------- .../project/partials/FilterInput.vue | 32 ++++++++++++++++--- 2 files changed, 27 insertions(+), 33 deletions(-) diff --git a/frontend/src/components/input/AutocompleteDropdown.vue b/frontend/src/components/input/AutocompleteDropdown.vue index 927d7fa14..e5eb018cd 100644 --- a/frontend/src/components/input/AutocompleteDropdown.vue +++ b/frontend/src/components/input/AutocompleteDropdown.vue @@ -27,30 +27,6 @@ watch( const emit = defineEmits(['blur']) -const placeholderText = computed(() => { - const value = (model.value || '').replace(/[\n\r\t]/gi, ' ') - - if (state.value === 'unfocused') { - return value ? '' : props.suggestion - } - - if (!value || !value.trim()) { - return props.suggestion - } - - return lookahead() -}) - -const spacerText = computed(() => { - const value = (model.value || '').replace(/[\n\r\t]/gi, ' ') - - if (!value || !value.trim()) { - return props.suggestion - } - - return value -}) - const props = withDefaults(defineProps<{ options: any[], suggestion?: string, @@ -161,14 +137,10 @@ function onUpdateField(e) {
-
{{ spacerText }}
-
{{ placeholderText }}