Skip to content

Commit 782a1cc

Browse files
zwwhdlszwwhdls
authored
Merge pull request #59 from basenana/fix/meilisearch
fix: split doc and attr into different index in meilisearch
2 parents bd105fc + e397d12 commit 782a1cc

File tree

4 files changed

+159
-110
lines changed

4 files changed

+159
-110
lines changed

‎pkg/models/doc/document.go

+22
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@ package doc
1919
import (
2020
"fmt"
2121
"time"
22+
23+
"github.com/basenana/friday/pkg/models"
24+
"github.com/basenana/friday/pkg/utils"
2225
)
2326

2427
type Document struct {
@@ -38,6 +41,25 @@ type Document struct {
3841
ChangedAt time.Time `json:"changed_at"`
3942
}
4043

44+
func (d *Document) NewTest() *Document {
45+
return &Document{
46+
EntryId: 1,
47+
Name: "test",
48+
Namespace: models.DefaultNamespaceValue,
49+
ParentEntryID: utils.ToPtr(int64(1)),
50+
Source: "test",
51+
Content: "test",
52+
Summary: "test",
53+
WebUrl: "test",
54+
HeaderImage: "test",
55+
SubContent: "test",
56+
Marked: utils.ToPtr(true),
57+
Unread: utils.ToPtr(true),
58+
CreatedAt: time.Now(),
59+
ChangedAt: time.Now(),
60+
}
61+
}
62+
4163
type DocumentFilter struct {
4264
Namespace string
4365
Search string

‎pkg/store/meili/meili.go

+68-61
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121
"encoding/json"
2222
"fmt"
23+
"strings"
2324
"time"
2425

2526
"github.com/meilisearch/meilisearch-go"
@@ -66,95 +67,88 @@ func NewMeiliClient(conf config.Config) (store.DocStoreInterface, error) {
6667
}
6768

6869
func (c *Client) init() error {
69-
attrs, err := c.docIndex.GetFilterableAttributes()
70-
if err != nil {
70+
testDoc := (&doc.Document{}).NewTest()
71+
// doc index
72+
if err := c.initIndex(c.docIndex, DocFilterableAttrs, DocSortAttrs, func() error {
73+
return c.CreateDocument(context.TODO(), testDoc)
74+
}); err != nil {
7175
return err
7276
}
73-
if !utils.Equal(DocFilterableAttrs, attrs) {
74-
t, err := c.docIndex.UpdateFilterableAttributes(&DocFilterableAttrs)
75-
if err != nil {
76-
return err
77-
}
78-
if err = c.wait(context.TODO(), "document", t.TaskUID); err != nil {
79-
return err
80-
}
81-
}
8277

83-
sortAttrs := DocSortAttrs
84-
crtSortAttrs, err := c.docIndex.GetSortableAttributes()
85-
if err != nil {
78+
// attr index
79+
if err := c.initIndex(c.attrIndex, DocAttrFilterableAttrs, DocAttrSortAttrs, func() error {
80+
return c.CreateDocument(context.TODO(), testDoc)
81+
}); err != nil {
8682
return err
8783
}
88-
if !utils.Equal(sortAttrs, crtSortAttrs) {
89-
t, err := c.docIndex.UpdateSortableAttributes(&sortAttrs)
90-
if err != nil {
84+
85+
d, err := c.GetDocument(context.TODO(), testDoc.EntryId)
86+
if (err != nil && strings.Contains(err.Error(), "not found")) || (err == nil && d == nil) {
87+
return nil
88+
} else if d != nil {
89+
return c.DeleteDocument(context.TODO(), testDoc.EntryId)
90+
}
91+
return err
92+
}
93+
94+
func (c *Client) initIndex(index meilisearch.IndexManager, filterableAttrs, sortableAttrs []string, noIndexFn func() error) error {
95+
attrs, err := index.GetFilterableAttributes()
96+
if err != nil {
97+
if !strings.Contains(err.Error(), "not found") {
9198
return err
9299
}
93-
if err = c.wait(context.TODO(), "document", t.TaskUID); err != nil {
100+
if err := noIndexFn(); err != nil {
94101
return err
95102
}
96103
}
97-
98-
// attr index
99-
attrAttrs, err := c.attrIndex.GetFilterableAttributes()
100-
if err != nil {
101-
return err
102-
}
103-
if !utils.Equal(DocAttrFilterableAttrs, attrAttrs) {
104-
t, err := c.docIndex.UpdateFilterableAttributes(&DocAttrFilterableAttrs)
104+
if !utils.Equal(filterableAttrs, attrs) {
105+
t, err := index.UpdateFilterableAttributes(&filterableAttrs)
105106
if err != nil {
106107
return err
107108
}
108-
if err = c.wait(context.TODO(), "attr", t.TaskUID); err != nil {
109+
if err = c.wait(context.TODO(), index, t.TaskUID); err != nil {
109110
return err
110111
}
111112
}
112-
attrSortAttrs := DocAttrSortAttrs
113-
crtAttrSortAttrs, err := c.docIndex.GetSortableAttributes()
113+
114+
crtSortAttrs, err := index.GetSortableAttributes()
114115
if err != nil {
115116
return err
116117
}
117-
if !utils.Equal(attrSortAttrs, crtAttrSortAttrs) {
118-
t, err := c.docIndex.UpdateSortableAttributes(&attrSortAttrs)
118+
if !utils.Equal(sortableAttrs, crtSortAttrs) {
119+
t, err := index.UpdateSortableAttributes(&sortableAttrs)
119120
if err != nil {
120121
return err
121122
}
122-
if err = c.wait(context.TODO(), "attr", t.TaskUID); err != nil {
123+
if err = c.wait(context.TODO(), index, t.TaskUID); err != nil {
123124
return err
124125
}
125126
}
126127
return nil
127128
}
128129

129-
func (c *Client) index(kind string) meilisearch.IndexManager {
130-
if kind == "attr" {
131-
return c.attrIndex
132-
}
133-
return c.docIndex
134-
}
135-
136130
func (c *Client) CreateDocument(ctx context.Context, doc *doc.Document) error {
137131
newDoc := (&Document{}).FromModel(doc)
138132
c.log.Debugf("store entryId %s", newDoc.EntryId)
139-
task, err := c.index(newDoc.Kind).AddDocuments(newDoc, "id")
133+
task, err := c.docIndex.AddDocuments(newDoc, "id")
140134
if err != nil {
141135
c.log.Error(err)
142136
return err
143137
}
144-
if err := c.wait(ctx, newDoc.Kind, task.TaskUID); err != nil {
138+
if err := c.wait(ctx, c.docIndex, task.TaskUID); err != nil {
145139
c.log.Errorf("store document with entryId %s error: %s", newDoc.EntryId, err)
146140
return err
147141
}
148142

149143
// store document attr
150144
newAttrs := (&DocumentAttrList{}).FromModel(doc)
151145
c.log.Debugf("store doc of entryId %d attrs: %s", doc.EntryId, newAttrs.String())
152-
t, err := c.index("attr").AddDocuments(newAttrs, "id")
146+
t, err := c.attrIndex.AddDocuments(newAttrs, "id")
153147
if err != nil {
154148
c.log.Error(err)
155149
return err
156150
}
157-
if err := c.wait(ctx, "attr", t.TaskUID); err != nil {
151+
if err := c.wait(ctx, c.attrIndex, t.TaskUID); err != nil {
158152
c.log.Errorf("store document attr of entryId %d error: %s", doc.EntryId, err)
159153
return err
160154
}
@@ -170,24 +164,24 @@ func (c *Client) UpdateDocument(ctx context.Context, doc *doc.Document) error {
170164
for _, aq := range newAttrsQuery.AttrQueries {
171165
filter = append(filter, aq.ToFilter())
172166
}
173-
t, err := c.index("attr").DeleteDocumentsByFilter(filter)
167+
t, err := c.attrIndex.DeleteDocumentsByFilter(filter)
174168
if err != nil {
175169
c.log.Error(err)
176170
return err
177171
}
178-
if err = c.wait(ctx, "attr", t.TaskUID); err != nil {
172+
if err = c.wait(ctx, c.attrIndex, t.TaskUID); err != nil {
179173
c.log.Errorf("delete document by filter error: %s", err)
180174
return err
181175
}
182176
// store document attr
183177
newAttrs := (&DocumentAttrList{}).FromModel(doc)
184178
c.log.Debugf("store doc of entryId %d attrs: %s", doc.EntryId, newAttrs.String())
185-
t, err = c.index("attr").AddDocuments(newAttrs, "id")
179+
t, err = c.attrIndex.AddDocuments(newAttrs, "id")
186180
if err != nil {
187181
c.log.Error(err)
188182
return err
189183
}
190-
if err := c.wait(ctx, "attr", t.TaskUID); err != nil {
184+
if err := c.wait(ctx, c.attrIndex, t.TaskUID); err != nil {
191185
c.log.Errorf("store document attr of entryId %d error: %s", doc.EntryId, err)
192186
return err
193187
}
@@ -198,7 +192,7 @@ func (c *Client) GetDocument(ctx context.Context, entryId int64) (*doc.Document,
198192
namespace := models.GetNamespace(ctx)
199193
query := (&DocumentQuery{}).OfEntryId(namespace.String(), entryId)
200194
c.log.Debugf("get document by entryId: %d", entryId)
201-
rep, err := c.index("document").Search("", query.ToRequest())
195+
rep, err := c.docIndex.Search("", query.ToRequest())
202196
if err != nil {
203197
return nil, err
204198
}
@@ -215,7 +209,7 @@ func (c *Client) GetDocument(ctx context.Context, entryId int64) (*doc.Document,
215209
// get attrs
216210
attrQuery := (&DocumentAttrQuery{}).OfEntryId(document.Namespace, document.EntryId)
217211
c.log.Debugf("filter document attr: %s", attrQuery.String())
218-
attrRep, err := c.index("attr").Search("", attrQuery.ToRequest())
212+
attrRep, err := c.attrIndex.Search("", attrQuery.ToRequest())
219213
if err != nil {
220214
return nil, err
221215
}
@@ -241,7 +235,7 @@ func (c *Client) FilterDocuments(ctx context.Context, filter *doc.DocumentFilter
241235
attrQuery := (&DocumentAttrQueries{}).FromFilter(filter)
242236
for _, aq := range *attrQuery {
243237
c.log.Debugf("filter document attr: %s", aq.String())
244-
attrRep, err := c.index("attr").Search("", aq.ToRequest())
238+
attrRep, err := c.attrIndex.Search("", aq.ToRequest())
245239
if err != nil {
246240
return nil, err
247241
}
@@ -263,11 +257,14 @@ func (c *Client) FilterDocuments(ctx context.Context, filter *doc.DocumentFilter
263257
Option: "IN",
264258
Value: entryIds,
265259
})
260+
} else {
261+
// no result
262+
return nil, nil
266263
}
267264
}
268265

269266
c.log.Debugf("search document: [%s] query: [%s]", query.Search, query.String())
270-
rep, err := c.index("document").Search(query.Search, query.ToRequest())
267+
rep, err := c.docIndex.Search(query.Search, query.ToRequest())
271268
if err != nil {
272269
return nil, err
273270
}
@@ -282,16 +279,17 @@ func (c *Client) FilterDocuments(ctx context.Context, filter *doc.DocumentFilter
282279
c.log.Errorf("unmarshal document error: %s", err)
283280
continue
284281
}
282+
c.log.Debugf("get document: %s", document.String())
285283

286284
// get attrs
287285
attrQuery := (&DocumentAttrQuery{}).OfEntryId(document.Namespace, document.EntryId)
288286
c.log.Debugf("filter document attr: %s", attrQuery.String())
289-
attrRep, err := c.index("attr").Search("", attrQuery.ToRequest())
287+
attrRep, err := c.attrIndex.Search("", attrQuery.ToRequest())
290288
if err != nil {
291289
return nil, err
292290
}
293291

294-
attrs := make([]*DocumentAttr, 0)
292+
attrs := DocumentAttrList{}
295293
for _, hit := range attrRep.Hits {
296294
b, _ := json.Marshal(hit)
297295
attr := &DocumentAttr{}
@@ -302,38 +300,47 @@ func (c *Client) FilterDocuments(ctx context.Context, filter *doc.DocumentFilter
302300
}
303301
attrs = append(attrs, attr)
304302
}
303+
c.log.Debugf("filter [%d] document attr: %s", len(attrs), attrs.String())
305304
documents = append(documents, document.ToModel(attrs))
306305
}
307306
return documents, nil
308307
}
309308

310309
func (c *Client) DeleteDocument(ctx context.Context, entryId int64) error {
311310
c.log.Debugf("delete document by entryId: %d", entryId)
312-
aq := &AttrQuery{
313-
Attr: "entryId",
314-
Option: "=",
315-
Value: fmt.Sprintf("%d", entryId),
311+
ns := models.GetNamespace(ctx)
312+
dq := (&DocumentQuery{}).OfEntryId(ns.String(), entryId)
313+
t, err := c.docIndex.DeleteDocumentsByFilter(dq.ToFilter())
314+
if err != nil {
315+
c.log.Error(err)
316+
return err
316317
}
317-
t, err := c.index("attr").DeleteDocumentsByFilter(aq.ToFilter())
318+
if err := c.wait(ctx, c.docIndex, t.TaskUID); err != nil {
319+
c.log.Errorf("delete document by filter error: %s", err)
320+
}
321+
322+
c.log.Debugf("delete document attr by entryId: %d", entryId)
323+
aq := (&DocumentAttrQuery{}).OfEntryId(ns.String(), fmt.Sprintf("%d", entryId))
324+
t, err = c.attrIndex.DeleteDocumentsByFilter(aq.ToFilter())
318325
if err != nil {
319326
c.log.Error(err)
320327
return err
321328
}
322-
if err := c.wait(ctx, "attr", t.TaskUID); err != nil {
329+
if err := c.wait(ctx, c.attrIndex, t.TaskUID); err != nil {
323330
c.log.Errorf("delete document by filter error: %s", err)
324331
}
325332
return nil
326333
}
327334

328-
func (c *Client) wait(ctx context.Context, kind string, taskUID int64) error {
335+
func (c *Client) wait(ctx context.Context, index meilisearch.IndexManager, taskUID int64) error {
329336
t := time.NewTicker(100 * time.Millisecond)
330337
defer t.Stop()
331338
for {
332339
select {
333340
case <-ctx.Done():
334341
return fmt.Errorf("context timeout")
335342
case <-t.C:
336-
t, err := c.index(kind).GetTask(taskUID)
343+
t, err := index.GetTask(taskUID)
337344
if err != nil {
338345
c.log.Error(err)
339346
return err

0 commit comments

Comments
 (0)