ami/tl-vikunja
1
0
Fork 0
Code Activity

feat: fetch all projects with a recursive cte instead of recursive query

Browse Source 
This change modifies the fetching of all projects to use a recursive common table expression instead of recursively calling the method.
This commit is contained in:
kolaente
2024-02-28 12:02:12 +01:00
parent 5d127c2897
commit 6b1e67485b
6 changed files with 57 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
pkg/models/project.go
View File

@ -17,6 +17,7 @@
package models
import (
"fmt"
"math"
"strconv"
"strings"
@ -418,67 +419,53 @@ func getUserProjectsStatement(parentProjectIDs []int64, userID int64, search str
parentCondition,
builder.NotIn("l.id", parentProjectIDs),
)).
OrderBy("position").
GroupBy("l.id")
}
func getAllProjectsForUser(s *xorm.Session, userID int64, parentProjectIDs []int64, opts *projectOptions, projects *[]*Project, oldTotalCount int64, archivedProjects map[int64]bool) (resultCount int, totalCount int64, err error) {
func getAllProjectsForUser(s *xorm.Session, userID int64, opts *projectOptions) (projects []*Project, totalCount int64, err error) {
limit, start := getLimitFromPageIndex(opts.page, opts.perPage)
query := getUserProjectsStatement(parentProjectIDs, userID, opts.search, opts.getArchived)
if limit > 0 {
query = query.Limit(limit, start)
query := getUserProjectsStatement(nil, userID, opts.search, opts.getArchived)
querySQLString, args, err := query.ToSQL()
if err != nil {
return nil, 0, err
}
var limitSQL string
if limit > 0 {
limitSQL = fmt.Sprintf("LIMIT %d OFFSET %d", limit, start)
}
baseQuery := querySQLString + `
UNION ALL
SELECT p.* FROM projects p
INNER JOIN all_projects ap ON p.parent_project_id = ap.id`
currentProjects := []*Project{}
err = s.SQL(query).Find(&currentProjects)
err = s.SQL(`WITH RECURSIVE all_projects as (
`+baseQuery+`
ORDER BY position
`+limitSQL+`
)
SELECT * FROM all_projects GROUP BY all_projects.id ORDER BY position`, args...).Find(&currentProjects)
if err != nil {
return 0, 0, err
return
}
if len(currentProjects) == 0 {
return 0, oldTotalCount, err
return nil, 0, err
}
query = getUserProjectsStatement(parentProjectIDs, userID, opts.search, opts.getArchived)
totalCount, err = s.
SQL(query.Select("count(*)")).
SQL(`WITH RECURSIVE all_projects as (`+baseQuery+`)
SELECT count(*) FROM all_projects`, args...).
Count(&Project{})
if err != nil {
return 0, 0, err
return nil, 0, err
}
parentIDsMap := make(map[int64]bool, len(parentProjectIDs))
for _, id := range parentProjectIDs {
parentIDsMap[id] = true
}
for _, project := range currentProjects {
parentIDsMap[project.ID] = true
}
newParentIDs := []int64{}
for _, project := range currentProjects {
if project.IsArchived {
archivedProjects[project.ID] = true
}
if archivedProjects[project.ParentProjectID] {
project.IsArchived = true
}
// Filter out parent project ids which we're not looking for to avoid leaking
// information about parent projects
if !parentIDsMap[project.ParentProjectID] {
project.ParentProjectID = 0
}
newParentIDs = append(newParentIDs, project.ID)
}
*projects = append(*projects, currentProjects...)
// If we don't reset the limit for subprojects, it will be impossible to fetch all subprojects.
opts.page = -1
return getAllProjectsForUser(s, userID, newParentIDs, opts, projects, oldTotalCount+totalCount, archivedProjects)
return currentProjects, totalCount, err
}
// Gets the projects with their children without any tasks
@ -488,9 +475,7 @@ func getRawProjectsForUser(s *xorm.Session, opts *projectOptions) (projects []*P
return nil, 0, 0, err
}
allProjects := []*Project{}
archivedProjects := make(map[int64]bool)
resultCount, totalItems, err = getAllProjectsForUser(s, fullUser.ID, nil, opts, &allProjects, 0, archivedProjects)
allProjects, totalItems, err := getAllProjectsForUser(s, fullUser.ID, opts)
if err != nil {
return
}

12
pkg/models/project_test.go
View File

@ -349,11 +349,9 @@ func TestProject_ReadAll(t *testing.T) {
t.Run("all", func(t *testing.T) {
db.LoadAndAssertFixtures(t)
s := db.NewSession()
projects := []*Project{}
archivedProjects := make(map[int64]bool)
_, _, err := getAllProjectsForUser(s, 1, nil, &projectOptions{}, &projects, 0, archivedProjects)
projects, _, err := getAllProjectsForUser(s, 6, &projectOptions{})
require.NoError(t, err)
assert.Len(t, projects, 24)
assert.Len(t, projects, 25)
_ = s.Close()
})
t.Run("only child projects for one project", func(t *testing.T) {
@ -369,12 +367,12 @@ func TestProject_ReadAll(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, reflect.Slice, reflect.TypeOf(projects3).Kind())
ls := projects3.([]*Project)
assert.Len(t, ls, 26)
assert.Len(t, ls, 28)
assert.Equal(t, int64(3), ls[0].ID) // Project 3 has a position of 1 and should be sorted first
assert.Equal(t, int64(1), ls[1].ID)
assert.Equal(t, int64(6), ls[2].ID)
assert.Equal(t, int64(-1), ls[24].ID)
assert.Equal(t, int64(-2), ls[25].ID)
assert.Equal(t, int64(-1), ls[26].ID)
assert.Equal(t, int64(-2), ls[27].ID)
_ = s.Close()
})
t.Run("projects for nonexistant user", func(t *testing.T) {

17
pkg/models/task_collection_test.go
View File

@ -656,6 +656,18 @@ func TestTaskCollection_ReadAll(t *testing.T) {
Created: time.Unix(1543626724, 0).In(loc),
Updated: time.Unix(1543626724, 0).In(loc),
}
task39 := &Task{
ID: 39,
Title: "task #39",
Identifier: "#0",
CreatedByID: 1,
CreatedBy: user1,
ProjectID: 25,
RelatedTasks: map[RelationKind][]*Task{},
BucketID: 0,
Created: time.Unix(1543626724, 0).In(loc),
Updated: time.Unix(1543626724, 0).In(loc),
}
type fields struct {
ProjectID int64
@ -728,6 +740,7 @@ func TestTaskCollection_ReadAll(t *testing.T) {
task32,
task33,
task35,
task39,
},
wantErr: false,
},
@ -772,6 +785,7 @@ func TestTaskCollection_ReadAll(t *testing.T) {
task3,
task1,
task2,
task39,
},
wantErr: false,
},
@ -943,6 +957,7 @@ func TestTaskCollection_ReadAll(t *testing.T) {
task32, // has nil dates
task33, // has nil dates
task35, // has nil dates
task39, // has nil dates
},
wantErr: false,
},
@ -1202,6 +1217,7 @@ func TestTaskCollection_ReadAll(t *testing.T) {
task32,
task33,
task35,
task39,
},
},
{
@ -1218,6 +1234,7 @@ func TestTaskCollection_ReadAll(t *testing.T) {
task6,
task5,
// The other ones don't have a due date
task39,
task35,
task33,
task32,