1
0

fix(typesense): use typesense bulk insert, log all errors

(cherry picked from commit 207b88a286dc9881034af9d9d117d67e8ae3c91f)
This commit is contained in:
kolaente 2024-09-20 10:14:41 +02:00
parent 940d149e2a
commit a0ad6e7d28
No known key found for this signature in database
GPG Key ID: F40E70337AB24C9B

View File

@ -296,6 +296,7 @@ func reindexTasksInTypesense(s *xorm.Session, tasks map[int64]*Task) (err error)
} }
projects := make(map[int64]*Project) projects := make(map[int64]*Project)
typesenseTasks := []interface{}{}
positionsByTask, err := getPositionsByTask(s) positionsByTask, err := getPositionsByTask(s)
if err != nil { if err != nil {
@ -307,7 +308,6 @@ func reindexTasksInTypesense(s *xorm.Session, tasks map[int64]*Task) (err error)
return err return err
} }
indexedTasks := 0
for _, task := range tasks { for _, task := range tasks {
ttask, err := getTypesenseTaskForTask(s, task, projects, positionsByTask, bucketsByTask) ttask, err := getTypesenseTaskForTask(s, task, projects, positionsByTask, bucketsByTask)
@ -319,17 +319,27 @@ func reindexTasksInTypesense(s *xorm.Session, tasks map[int64]*Task) (err error)
continue continue
} }
_, err = typesenseClient.Collection("tasks"). typesenseTasks = append(typesenseTasks, ttask)
Documents().
Upsert(context.Background(), ttask)
if err != nil {
log.Errorf("Could not upsert task into Typesense: %s", err)
return err
}
indexedTasks++
} }
log.Debugf("Indexed %d tasks into Typesense", indexedTasks) response, err := typesenseClient.Collection("tasks").
Documents().
Import(context.Background(), typesenseTasks, &api.ImportDocumentsParams{
Action: pointer.String("emplace"),
BatchSize: pointer.Int(100),
})
if err != nil {
log.Errorf("Could not upsert tasks into Typesense: %s", err)
return err
}
for _, r := range response {
if r.Success {
continue
}
log.Errorf("Errors during index: [error=%s, document=%s]", r.Error, r.Document)
}
log.Debugf("Indexed %d tasks into Typesense", len(typesenseTasks))
return nil return nil
} }