mirror of
https://github.com/oxynote/oxynote.git
synced 2026-10-17 02:49:18 +02:00
add group search
This commit is contained in:
david
parent
97b06efd78
commit
e670cb6cb4
|
|
@ -38,27 +38,25 @@ require (
|
|||
github.com/jarcoal/httpmock v1.4.2 // indirect
|
||||
github.com/lib/pq v1.10.9 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/moby/moby/api v1.55.0 // indirect
|
||||
github.com/moby/moby/client v0.5.1 // indirect
|
||||
github.com/moby/moby/api v1.56.0 // indirect
|
||||
github.com/moby/moby/client v0.6.0 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/prometheus/common v0.71.0 // indirect
|
||||
github.com/prometheus/common v0.72.0 // indirect
|
||||
github.com/prometheus/procfs v0.21.1 // indirect
|
||||
github.com/robfig/cron/v3 v3.0.1 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 // indirect
|
||||
go.opentelemetry.io/otel v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.46.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.28.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/sync v0.22.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/text v0.41.0 // indirect
|
||||
golang.org/x/sync v0.23.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/text v0.42.0 // indirect
|
||||
google.golang.org/protobuf v1.36.12 // indirect
|
||||
)
|
||||
|
|
|
|||
|
|
@ -50,8 +50,8 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo
|
|||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/jarcoal/httpmock v1.4.2 h1:dKwiP/9zITCPfBLsDn3kchbSOu16JrnxtVEmL0fPRcI=
|
||||
github.com/jarcoal/httpmock v1.4.2/go.mod h1:ftW1xULwo+j0R0JJkJIIi7UKigZUXCLLanykgjwBXL0=
|
||||
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
|
||||
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
|
|
@ -60,10 +60,10 @@ github.com/maxatome/go-testdeep v1.14.0 h1:rRlLv1+kI8eOI3OaBXZwb3O7xY3exRzdW5QyX
|
|||
github.com/maxatome/go-testdeep v1.14.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM=
|
||||
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
||||
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
|
||||
github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc=
|
||||
github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
|
||||
github.com/moby/moby/client v0.5.1 h1:tYNaJno4c0HXz12y5BiqEDy0rVTYkWzI26lGvnTMiJw=
|
||||
github.com/moby/moby/client v0.5.1/go.mod h1:odLstlZ6uSnfvAgVxMpvgmb8SUdd+siH2T0GBuxVAlM=
|
||||
github.com/moby/moby/api v1.56.0 h1:GQzua3NA599ASSIICx0iFgiJeO9YkdDARvQsm23ZZuQ=
|
||||
github.com/moby/moby/api v1.56.0/go.mod h1:sZ+THbVWkjOmBPPfbnzdD/G1LuIexWhqlSHHPTDQ1Uk=
|
||||
github.com/moby/moby/client v0.6.0 h1:AJjEB21QPbXSXjDsZorFBoDZPhMrfbpaPLgSMAW9Bgs=
|
||||
github.com/moby/moby/client v0.6.0/go.mod h1:OCo00wNRyA3m4lmJ228W3JbyCN4ZNNYjpOXiJydBdcQ=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
|
|
@ -81,8 +81,8 @@ github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59u
|
|||
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
|
||||
github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo=
|
||||
github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM=
|
||||
github.com/prometheus/common v0.71.0 h1:9KDAKb7Mj3HEVKyFCK6Dc/HIwlBzZIN2l7/lrHl3KK8=
|
||||
github.com/prometheus/common v0.71.0/go.mod h1:CLJ5H8TEsGX8bl31BdMkfhIZ+QmZ9tBPPotUxUbfcmk=
|
||||
github.com/prometheus/common v0.72.0 h1:tAYsE+sPJxIncDAobm4H5aQjmox9ZxEIIqPbiffa8G4=
|
||||
github.com/prometheus/common v0.72.0/go.mod h1:77NWqAQ2tXT7BIK40qjJdw5Acrsrg1TlHAnsQi3i6mk=
|
||||
github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
|
||||
github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
|
|
@ -96,8 +96,8 @@ github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWD
|
|||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU=
|
||||
go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc=
|
||||
go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE=
|
||||
go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8=
|
||||
|
|
@ -118,12 +118,12 @@ go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
|||
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
|
||||
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
|
||||
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
|
||||
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
|
||||
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
|
||||
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
|
||||
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
|
|
|
|||
139
server/core/internal/search/branch.go
Normal file
139
server/core/internal/search/branch.go
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
package search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"strconv"
|
||||
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
bleveSearch "github.com/blevesearch/bleve/v2/search"
|
||||
"github.com/guregu/null/v5"
|
||||
"github.com/rs/xid"
|
||||
)
|
||||
|
||||
const (
|
||||
// BranchLimitDefault is the number of hits a branch page holds when
|
||||
// the request names no limit.
|
||||
BranchLimitDefault = 20
|
||||
|
||||
// BranchLimitMax caps the number of hits one branch page holds.
|
||||
BranchLimitMax = 50
|
||||
)
|
||||
|
||||
// BranchQuery describes one page of the hits of one branch.
|
||||
type BranchQuery struct {
|
||||
// OrganizationID specifies the organization searched.
|
||||
OrganizationID string
|
||||
|
||||
// BranchID specifies the branch searched.
|
||||
BranchID xid.ID
|
||||
|
||||
// Query specifies the text searched for.
|
||||
Query string
|
||||
|
||||
// Limit specifies the number of hits on the page, see BranchLimit.
|
||||
Limit int
|
||||
|
||||
// PageToken specifies the page, empty for the first one. A group's
|
||||
// NextHitsToken continues after the hits the group shows.
|
||||
PageToken string
|
||||
}
|
||||
|
||||
// fingerprint identifies the search a page token belongs to. A token
|
||||
// replayed with another query or branch would point into another order.
|
||||
func (bq BranchQuery) fingerprint() string {
|
||||
h := fnv.New64a()
|
||||
h.Write([]byte(bq.Query + "\x00" + bq.BranchID.String())) //nolint:gosec // hash writes never fail
|
||||
|
||||
return strconv.FormatUint(h.Sum64(), 36)
|
||||
}
|
||||
|
||||
// BranchPage is one page of the hits of one branch.
|
||||
type BranchPage struct {
|
||||
// Hits specifies the page's matching content blocks, best first.
|
||||
// Their text is HTML with the matched terms wrapped in mark tags.
|
||||
Hits []Block
|
||||
|
||||
// TotalHits specifies the number of matching content blocks across
|
||||
// every page.
|
||||
TotalHits int
|
||||
|
||||
// NextPageToken specifies the token of the next page, if any.
|
||||
NextPageToken null.String
|
||||
}
|
||||
|
||||
// SearchBranch searches one branch's content entries and returns one page
|
||||
// of them, highlighted. The hits come in the order SearchGroups shows a
|
||||
// group's hits in, so a group's NextHitsToken continues the group.
|
||||
func (i *Index) SearchBranch(ctx context.Context, bq BranchQuery) (BranchPage, error) {
|
||||
if err := ValidateQuery(bq.Query); err != nil {
|
||||
return BranchPage{}, err
|
||||
}
|
||||
|
||||
fingerprint := bq.fingerprint()
|
||||
|
||||
offset, err := decodePageToken(bq.PageToken, fingerprint)
|
||||
if err != nil {
|
||||
return BranchPage{}, err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, _searchTimeout)
|
||||
defer cancel()
|
||||
|
||||
ctx = context.WithValue(ctx, bleveSearch.MaxTermSearchersKey, _maxTermSearchers)
|
||||
|
||||
branch := bleve.NewTermQuery(bq.BranchID.String())
|
||||
branch.SetField(_fieldBranchID)
|
||||
|
||||
// a document name is a group's title, never one of its hits.
|
||||
name := bleve.NewTermQuery("document")
|
||||
name.SetField(_fieldType)
|
||||
|
||||
q := bleve.NewBooleanQuery()
|
||||
q.AddMust(i.query(bq.OrganizationID, bq.Query), branch)
|
||||
q.AddMustNot(name)
|
||||
|
||||
limit := BranchLimit(bq.Limit)
|
||||
|
||||
req := bleve.NewSearchRequestOptions(q, limit, offset, false)
|
||||
req.Fields = []string{"*"}
|
||||
req.SortBy([]string{"-_score", "_id"})
|
||||
req.Highlight = bleve.NewHighlightWithStyle(_highlighterName)
|
||||
req.Highlight.AddField(_fieldText)
|
||||
|
||||
res, err := i.idx.SearchInContext(ctx, req)
|
||||
if err != nil {
|
||||
return BranchPage{}, fmt.Errorf("searching branch: %w", err)
|
||||
}
|
||||
|
||||
page := BranchPage{
|
||||
Hits: make([]Block, 0, len(res.Hits)),
|
||||
TotalHits: int(res.Total),
|
||||
}
|
||||
|
||||
for _, hit := range res.Hits {
|
||||
b, err := i.decodeHighlightedHit(hit)
|
||||
if err != nil {
|
||||
return BranchPage{}, err
|
||||
}
|
||||
|
||||
page.Hits = append(page.Hits, b)
|
||||
}
|
||||
|
||||
if end := offset + len(res.Hits); end < page.TotalHits {
|
||||
page.NextPageToken = null.StringFrom(encodePageToken(end, fingerprint))
|
||||
}
|
||||
|
||||
return page, nil
|
||||
}
|
||||
|
||||
// BranchLimit returns the page size for the requested limit: the default
|
||||
// when none is set, and at most BranchLimitMax.
|
||||
func BranchLimit(limit int) int {
|
||||
if limit <= 0 {
|
||||
return BranchLimitDefault
|
||||
}
|
||||
|
||||
return min(limit, BranchLimitMax)
|
||||
}
|
||||
250
server/core/internal/search/branch_test.go
Normal file
250
server/core/internal/search/branch_test.go
Normal file
|
|
@ -0,0 +1,250 @@
|
|||
package search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/guregu/null/v5"
|
||||
"github.com/oxynote/oxynote/server/core/pkg/testutil"
|
||||
"github.com/rs/xid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func Test_BranchQuery_fingerprint(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
bq := BranchQuery{
|
||||
OrganizationID: "org-1",
|
||||
BranchID: xid.New(),
|
||||
Query: "shipment",
|
||||
Limit: 5,
|
||||
PageToken: "token",
|
||||
}
|
||||
|
||||
branch := bq
|
||||
branch.BranchID = xid.New()
|
||||
|
||||
other := bq
|
||||
other.Query = "shipments"
|
||||
|
||||
// the organization, limit and token do not change the order a token
|
||||
// points into.
|
||||
same := bq
|
||||
same.OrganizationID = "org-2"
|
||||
same.Limit = 10
|
||||
same.PageToken = ""
|
||||
|
||||
assert.NotEmpty(t, bq.fingerprint())
|
||||
assert.Equal(t, bq.fingerprint(), same.fingerprint())
|
||||
assert.NotEqual(t, bq.fingerprint(), branch.fingerprint())
|
||||
assert.NotEqual(t, bq.fingerprint(), other.fingerprint())
|
||||
}
|
||||
|
||||
func Test_Index_SearchBranch(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
idx := openIndex(t, indexPath(t), &stubSource{})
|
||||
t.Cleanup(func() { require.NoError(t, idx.Close()) })
|
||||
|
||||
main := Scope{
|
||||
OrganizationID: "org-1",
|
||||
DocumentID: xid.New(),
|
||||
BranchID: xid.New(),
|
||||
BranchName: "main",
|
||||
BranchDefault: true,
|
||||
}
|
||||
|
||||
fork := main
|
||||
fork.BranchID = xid.New()
|
||||
fork.BranchName = "draft"
|
||||
fork.BranchDefault = false
|
||||
|
||||
other := Scope{
|
||||
OrganizationID: "org-2",
|
||||
DocumentID: xid.New(),
|
||||
BranchID: xid.New(),
|
||||
BranchName: "main",
|
||||
BranchDefault: true,
|
||||
}
|
||||
|
||||
for branchID, entries := range map[xid.ID]map[string]Block{
|
||||
main.BranchID: {
|
||||
"docname": main.Block("docname", "document", "Shipments"),
|
||||
"h1": main.Block("h1", "heading", "Shipment tracking"),
|
||||
"p1": main.Block("p1", "paragraph", "a shipment is created"),
|
||||
"p2": main.Block("p2", "paragraph", "every shipment has a label & a barcode"),
|
||||
"p3": main.Block("p3", "paragraph", "each shipment arrives with a shipment note"),
|
||||
"p4": main.Block("p4", "paragraph", "unrelated text"),
|
||||
},
|
||||
fork.BranchID: {
|
||||
"p1": fork.Block("p1", "paragraph", "a shipment is created"),
|
||||
},
|
||||
other.BranchID: {
|
||||
"p1": other.Block("p1", "paragraph", "shipment"),
|
||||
},
|
||||
} {
|
||||
require.NoError(t, idx.ReplaceBranch(context.Background(), branchID, entries))
|
||||
}
|
||||
|
||||
query := BranchQuery{
|
||||
OrganizationID: "org-1",
|
||||
BranchID: main.BranchID,
|
||||
Query: "shipment",
|
||||
}
|
||||
|
||||
all, err := idx.SearchBranch(context.Background(), query)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, all.Hits, 4)
|
||||
|
||||
// with returns the query with the limit and the page token of the
|
||||
// offset.
|
||||
with := func(limit, offset int) BranchQuery {
|
||||
bq := query
|
||||
bq.Limit = limit
|
||||
|
||||
if offset != 0 {
|
||||
bq.PageToken = encodePageToken(offset, query.fingerprint())
|
||||
}
|
||||
|
||||
return bq
|
||||
}
|
||||
|
||||
cancelled, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
type tcase struct {
|
||||
Context context.Context
|
||||
Query BranchQuery
|
||||
Result BranchPage
|
||||
Err error
|
||||
}
|
||||
|
||||
cc := map[string]tcase{
|
||||
"Invalid query": {
|
||||
Query: BranchQuery{OrganizationID: "org-1", BranchID: main.BranchID},
|
||||
Err: ErrInvalidQuery,
|
||||
},
|
||||
"Invalid page token": {
|
||||
Query: BranchQuery{OrganizationID: "org-1", BranchID: main.BranchID, Query: "shipment", PageToken: "%%%"},
|
||||
Err: ErrInvalidPageToken,
|
||||
},
|
||||
"Page token of another branch": func() tcase {
|
||||
bq := query
|
||||
bq.PageToken = encodePageToken(1, BranchQuery{BranchID: fork.BranchID, Query: "shipment"}.fingerprint())
|
||||
|
||||
return tcase{Query: bq, Err: ErrInvalidPageToken}
|
||||
}(),
|
||||
"Error returned by the index": {
|
||||
Context: cancelled,
|
||||
Query: query,
|
||||
Err: assert.AnError,
|
||||
},
|
||||
"No matches": func() tcase {
|
||||
bq := query
|
||||
bq.Query = "zebra"
|
||||
|
||||
return tcase{Query: bq, Result: BranchPage{Hits: []Block{}}}
|
||||
}(),
|
||||
"Entries of another organization are never returned": func() tcase {
|
||||
bq := query
|
||||
bq.BranchID = other.BranchID
|
||||
|
||||
return tcase{Query: bq, Result: BranchPage{Hits: []Block{}}}
|
||||
}(),
|
||||
"First page": {
|
||||
Query: with(2, 0),
|
||||
Result: BranchPage{
|
||||
Hits: all.Hits[:2],
|
||||
TotalHits: 4,
|
||||
NextPageToken: null.StringFrom(encodePageToken(2, query.fingerprint())),
|
||||
},
|
||||
},
|
||||
"Last page": {
|
||||
Query: with(2, 2),
|
||||
Result: BranchPage{
|
||||
Hits: all.Hits[2:],
|
||||
TotalHits: 4,
|
||||
},
|
||||
},
|
||||
"Page past the end": {
|
||||
Query: with(2, 10),
|
||||
Result: BranchPage{
|
||||
Hits: []Block{},
|
||||
TotalHits: 4,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for cn, c := range cc {
|
||||
t.Run(cn, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := c.Context
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
|
||||
res, err := idx.SearchBranch(ctx, c.Query)
|
||||
testutil.AssertEqualError(t, c.Err, err)
|
||||
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
assert.Equal(t, c.Result, res)
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("Continues the hits of a group", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
gp, err := idx.SearchGroups(context.Background(), GroupQuery{
|
||||
OrganizationID: "org-1",
|
||||
Query: "shipment",
|
||||
HitsLimit: 2,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
var group Group
|
||||
|
||||
for _, g := range gp.Groups {
|
||||
if g.BranchID == main.BranchID {
|
||||
group = g
|
||||
}
|
||||
}
|
||||
|
||||
require.True(t, group.NextHitsToken.Valid)
|
||||
|
||||
bq := query
|
||||
bq.PageToken = group.NextHitsToken.String
|
||||
|
||||
res, err := idx.SearchBranch(context.Background(), bq)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, all.Hits, append(group.Hits, res.Hits...))
|
||||
})
|
||||
}
|
||||
|
||||
func Test_BranchLimit(t *testing.T) {
|
||||
cc := map[string]struct {
|
||||
Limit int
|
||||
Result int
|
||||
}{
|
||||
"No limit": {Limit: 0, Result: BranchLimitDefault},
|
||||
"Negative limit": {Limit: -1, Result: BranchLimitDefault},
|
||||
"Limit within caps": {Limit: 5, Result: 5},
|
||||
"Limit over the cap": {
|
||||
Limit: BranchLimitMax + 1,
|
||||
Result: BranchLimitMax,
|
||||
},
|
||||
}
|
||||
|
||||
for cn, c := range cc {
|
||||
t.Run(cn, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assert.Equal(t, c.Result, BranchLimit(c.Limit))
|
||||
})
|
||||
}
|
||||
}
|
||||
461
server/core/internal/search/group.go
Normal file
461
server/core/internal/search/group.go
Normal file
|
|
@ -0,0 +1,461 @@
|
|||
package search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"html"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strconv"
|
||||
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
bleveSearch "github.com/blevesearch/bleve/v2/search"
|
||||
"github.com/blevesearch/bleve/v2/search/query"
|
||||
"github.com/guregu/null/v5"
|
||||
"github.com/oxynote/oxynote/server/core/internal/document"
|
||||
"github.com/oxynote/oxynote/server/core/pkg/errutil"
|
||||
"github.com/rs/xid"
|
||||
)
|
||||
|
||||
const (
|
||||
// GroupLimitDefault is the number of groups a page holds when the
|
||||
// request names no limit.
|
||||
GroupLimitDefault = 30
|
||||
|
||||
// GroupLimitMax caps the number of groups one page holds.
|
||||
GroupLimitMax = 50
|
||||
|
||||
// GroupHitsLimitDefault is the number of hits a group shows when the
|
||||
// request names no limit.
|
||||
GroupHitsLimitDefault = 3
|
||||
|
||||
// GroupHitsLimitMax caps the number of hits one group shows.
|
||||
GroupHitsLimitMax = 10
|
||||
|
||||
// _groupHitCap caps the hits one grouped search reads. The groups and
|
||||
// totals are built from these hits only.
|
||||
_groupHitCap = 1000
|
||||
)
|
||||
|
||||
// ErrInvalidPageToken is returned when a page token is malformed or was
|
||||
// issued for another search.
|
||||
var ErrInvalidPageToken = errutil.New(http.StatusBadRequest, "document.invalid_search_page_token", "invalid search page token")
|
||||
|
||||
// GroupQuery describes one page of a grouped search.
|
||||
type GroupQuery struct {
|
||||
// OrganizationID specifies the organization searched.
|
||||
OrganizationID string
|
||||
|
||||
// Query specifies the text searched for.
|
||||
Query string
|
||||
|
||||
// CurrentDocumentID specifies the document whose groups come first.
|
||||
CurrentDocumentID null.Value[xid.ID]
|
||||
|
||||
// Limit specifies the number of groups on the page, see GroupLimit.
|
||||
Limit int
|
||||
|
||||
// HitsLimit specifies the number of hits each group shows, see
|
||||
// GroupHitsLimit.
|
||||
HitsLimit int
|
||||
|
||||
// PageToken specifies the page, empty for the first one.
|
||||
PageToken string
|
||||
}
|
||||
|
||||
// fingerprint identifies the search a page token belongs to. A token
|
||||
// replayed with another query or pin would point into another order.
|
||||
func (gq GroupQuery) fingerprint() string {
|
||||
h := fnv.New64a()
|
||||
h.Write([]byte(gq.Query + "\x00" + gq.CurrentDocumentID.V.String())) //nolint:gosec // hash writes never fail
|
||||
|
||||
return strconv.FormatUint(h.Sum64(), 36)
|
||||
}
|
||||
|
||||
// Group is one branch of a document holding matches.
|
||||
type Group struct {
|
||||
// DocumentID specifies the document the branch belongs to.
|
||||
DocumentID xid.ID
|
||||
|
||||
// BranchID specifies the branch.
|
||||
BranchID xid.ID
|
||||
|
||||
// Title specifies the highlighted document name, set only when the
|
||||
// name matched.
|
||||
Title null.String
|
||||
|
||||
// Hits specifies the first matching content blocks, best first. Their
|
||||
// text is HTML with the matched terms wrapped in mark tags.
|
||||
Hits []Block
|
||||
|
||||
// TotalHits specifies the number of matching content blocks in the
|
||||
// branch, counted like GroupPage.TotalHits.
|
||||
TotalHits int
|
||||
|
||||
// NextHitsToken specifies the SearchBranch page token of the hits
|
||||
// after Hits, if any.
|
||||
NextHitsToken null.String
|
||||
}
|
||||
|
||||
// GroupPage is one page of a grouped search.
|
||||
type GroupPage struct {
|
||||
// Groups specifies the page's groups, best first.
|
||||
Groups []Group
|
||||
|
||||
// TotalHits specifies the number of matching content blocks across
|
||||
// every page.
|
||||
TotalHits int
|
||||
|
||||
// TotalDocuments specifies the number of groups across every page.
|
||||
TotalDocuments int
|
||||
|
||||
// Capped indicates that more matches exist than the totals count.
|
||||
Capped bool
|
||||
|
||||
// NextPageToken specifies the token of the next page, if any.
|
||||
NextPageToken null.String
|
||||
}
|
||||
|
||||
// BranchIDs returns the ids of the page's branches.
|
||||
func (gp GroupPage) BranchIDs() []xid.ID {
|
||||
ids := make([]xid.ID, 0, len(gp.Groups))
|
||||
|
||||
for _, g := range gp.Groups {
|
||||
ids = append(ids, g.BranchID)
|
||||
}
|
||||
|
||||
return ids
|
||||
}
|
||||
|
||||
// DataSourceIDs returns the ids of the data sources the page's hits
|
||||
// query, without repeats.
|
||||
func (gp GroupPage) DataSourceIDs() []string {
|
||||
var ids []string
|
||||
|
||||
for _, g := range gp.Groups {
|
||||
for _, h := range g.Hits {
|
||||
id := h.Attrs[document.AttrDataSourceID]
|
||||
if id != "" && !slices.Contains(ids, id) {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ids
|
||||
}
|
||||
|
||||
// pageToken is the decoded form of a page token.
|
||||
type pageToken struct {
|
||||
// Offset specifies the index of the page's first group or hit.
|
||||
Offset int `json:"o"`
|
||||
|
||||
// Fingerprint specifies the search the token belongs to.
|
||||
Fingerprint string `json:"f"`
|
||||
}
|
||||
|
||||
// groupEntries collects the ids of one group's matching entries.
|
||||
type groupEntries struct {
|
||||
// documentID specifies the document the branch belongs to.
|
||||
documentID xid.ID
|
||||
|
||||
// branchID specifies the branch.
|
||||
branchID xid.ID
|
||||
|
||||
// titleID specifies the id of the matching document name entry, if
|
||||
// any.
|
||||
titleID string
|
||||
|
||||
// hitIDs specifies the ids of the matching content entries, best
|
||||
// first.
|
||||
hitIDs []string
|
||||
}
|
||||
|
||||
// SearchGroups searches the organization's entries and returns one page
|
||||
// of them grouped by branch. Groups are ordered by their best match, with
|
||||
// the current document's groups first. A matching document name becomes
|
||||
// the group's title instead of a hit. Each group shows its first hits
|
||||
// only; SearchBranch pages through the rest.
|
||||
//
|
||||
// Grouping needs every match, which bleve cannot do in the index, so
|
||||
// the first _groupHitCap matches are read and grouped here on every
|
||||
// request. A page token is an offset into that order. Only the page's
|
||||
// own entries are then read in full and highlighted.
|
||||
func (i *Index) SearchGroups(ctx context.Context, gq GroupQuery) (GroupPage, error) {
|
||||
if err := ValidateQuery(gq.Query); err != nil {
|
||||
return GroupPage{}, err
|
||||
}
|
||||
|
||||
fingerprint := gq.fingerprint()
|
||||
|
||||
offset, err := decodePageToken(gq.PageToken, fingerprint)
|
||||
if err != nil {
|
||||
return GroupPage{}, err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, _searchTimeout)
|
||||
defer cancel()
|
||||
|
||||
ctx = context.WithValue(ctx, bleveSearch.MaxTermSearchersKey, _maxTermSearchers)
|
||||
|
||||
q := i.query(gq.OrganizationID, gq.Query)
|
||||
|
||||
req := bleve.NewSearchRequestOptions(q, _groupHitCap, 0, false)
|
||||
req.Fields = []string{_fieldDocumentID, _fieldBranchID, _fieldType}
|
||||
req.SortBy([]string{"-_score", "_id"})
|
||||
|
||||
res, err := i.idx.SearchInContext(ctx, req)
|
||||
if err != nil {
|
||||
return GroupPage{}, fmt.Errorf("searching documents: %w", err)
|
||||
}
|
||||
|
||||
groups, err := groupHits(res.Hits)
|
||||
if err != nil {
|
||||
return GroupPage{}, err
|
||||
}
|
||||
|
||||
if gq.CurrentDocumentID.Valid {
|
||||
// a stable sort keeps both parts in their match order.
|
||||
slices.SortStableFunc(groups, func(a, b *groupEntries) int {
|
||||
ac := a.documentID == gq.CurrentDocumentID.V
|
||||
bc := b.documentID == gq.CurrentDocumentID.V
|
||||
|
||||
switch {
|
||||
case ac && !bc:
|
||||
return -1
|
||||
case bc && !ac:
|
||||
return 1
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
page := GroupPage{
|
||||
TotalDocuments: len(groups),
|
||||
Capped: res.Total > uint64(len(res.Hits)),
|
||||
}
|
||||
|
||||
for _, g := range groups {
|
||||
page.TotalHits += len(g.hitIDs)
|
||||
}
|
||||
|
||||
limit := GroupLimit(gq.Limit)
|
||||
start := min(offset, len(groups))
|
||||
end := min(start+limit, len(groups))
|
||||
|
||||
if end < len(groups) {
|
||||
page.NextPageToken = null.StringFrom(encodePageToken(end, fingerprint))
|
||||
}
|
||||
|
||||
page.Groups, err = i.highlightGroups(ctx, gq, q, groups[start:end])
|
||||
if err != nil {
|
||||
return GroupPage{}, err
|
||||
}
|
||||
|
||||
return page, nil
|
||||
}
|
||||
|
||||
// highlightGroups reads the groups' title and first hit entries in full,
|
||||
// highlighted against the query. An entry removed since the groups were
|
||||
// built is left out.
|
||||
func (i *Index) highlightGroups(ctx context.Context, gq GroupQuery, q query.Query, groups []*groupEntries) ([]Group, error) {
|
||||
res := make([]Group, 0, len(groups))
|
||||
hitsLimit := GroupHitsLimit(gq.HitsLimit)
|
||||
|
||||
var ids []string
|
||||
|
||||
for _, g := range groups {
|
||||
if g.titleID != "" {
|
||||
ids = append(ids, g.titleID)
|
||||
}
|
||||
|
||||
ids = append(ids, g.hitIDs[:min(hitsLimit, len(g.hitIDs))]...)
|
||||
}
|
||||
|
||||
if len(ids) == 0 {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
req := bleve.NewSearchRequestOptions(
|
||||
bleve.NewConjunctionQuery(q, bleve.NewDocIDQuery(ids)),
|
||||
len(ids),
|
||||
0,
|
||||
false,
|
||||
)
|
||||
req.Fields = []string{"*"}
|
||||
req.Highlight = bleve.NewHighlightWithStyle(_highlighterName)
|
||||
req.Highlight.AddField(_fieldText)
|
||||
|
||||
sr, err := i.idx.SearchInContext(ctx, req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("highlighting documents: %w", err)
|
||||
}
|
||||
|
||||
blocks := make(map[string]Block, len(sr.Hits))
|
||||
|
||||
for _, hit := range sr.Hits {
|
||||
b, err := i.decodeHighlightedHit(hit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
blocks[hit.ID] = b
|
||||
}
|
||||
|
||||
for _, g := range groups {
|
||||
grp := Group{
|
||||
DocumentID: g.documentID,
|
||||
BranchID: g.branchID,
|
||||
Hits: []Block{},
|
||||
TotalHits: len(g.hitIDs),
|
||||
}
|
||||
|
||||
if b, ok := blocks[g.titleID]; ok {
|
||||
grp.Title = null.StringFrom(b.Text)
|
||||
}
|
||||
|
||||
for _, id := range g.hitIDs[:min(hitsLimit, len(g.hitIDs))] {
|
||||
if b, ok := blocks[id]; ok {
|
||||
grp.Hits = append(grp.Hits, b)
|
||||
}
|
||||
}
|
||||
|
||||
if len(g.hitIDs) > hitsLimit {
|
||||
bq := BranchQuery{
|
||||
BranchID: g.branchID,
|
||||
Query: gq.Query,
|
||||
}
|
||||
|
||||
grp.NextHitsToken = null.StringFrom(encodePageToken(hitsLimit, bq.fingerprint()))
|
||||
}
|
||||
|
||||
res = append(res, grp)
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// GroupLimit returns the page size for the requested limit: the default
|
||||
// when none is set, and at most GroupLimitMax.
|
||||
func GroupLimit(limit int) int {
|
||||
if limit <= 0 {
|
||||
return GroupLimitDefault
|
||||
}
|
||||
|
||||
return min(limit, GroupLimitMax)
|
||||
}
|
||||
|
||||
// GroupHitsLimit returns the number of hits a group shows for the
|
||||
// requested limit: the default when none is set, and at most
|
||||
// GroupHitsLimitMax.
|
||||
func GroupHitsLimit(limit int) int {
|
||||
if limit <= 0 {
|
||||
return GroupHitsLimitDefault
|
||||
}
|
||||
|
||||
return min(limit, GroupHitsLimitMax)
|
||||
}
|
||||
|
||||
// decodeHighlightedHit rebuilds an entry from a highlighted hit. Its text
|
||||
// is the best fragment, or the full text escaped like a fragment is when
|
||||
// the hit has none.
|
||||
func (i *Index) decodeHighlightedHit(hit *bleveSearch.DocumentMatch) (Block, error) {
|
||||
b, err := i.decodeHit(hit.ID, hit.Fields)
|
||||
if err != nil {
|
||||
return Block{}, err
|
||||
}
|
||||
|
||||
b.Text = html.EscapeString(b.Text)
|
||||
|
||||
if fragments := hit.Fragments[_fieldText]; len(fragments) != 0 {
|
||||
b.Text = fragments[0]
|
||||
}
|
||||
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// encodePageToken encodes the offset of a page of the search.
|
||||
func encodePageToken(offset int, fingerprint string) string {
|
||||
data, err := json.Marshal(pageToken{
|
||||
Offset: offset,
|
||||
Fingerprint: fingerprint,
|
||||
})
|
||||
if err != nil {
|
||||
// NOCOV: a struct of an int and a string always marshals.
|
||||
return ""
|
||||
}
|
||||
|
||||
return base64.RawURLEncoding.EncodeToString(data)
|
||||
}
|
||||
|
||||
// decodePageToken returns the offset a page token points at. An empty
|
||||
// token is the first page.
|
||||
func decodePageToken(token, fingerprint string) (int, error) {
|
||||
if token == "" {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
data, err := base64.RawURLEncoding.DecodeString(token)
|
||||
if err != nil {
|
||||
return 0, ErrInvalidPageToken
|
||||
}
|
||||
|
||||
var pt pageToken
|
||||
|
||||
if err := json.Unmarshal(data, &pt); err != nil {
|
||||
return 0, ErrInvalidPageToken
|
||||
}
|
||||
|
||||
if pt.Offset < 0 || pt.Fingerprint != fingerprint {
|
||||
return 0, ErrInvalidPageToken
|
||||
}
|
||||
|
||||
return pt.Offset, nil
|
||||
}
|
||||
|
||||
// groupHits groups the hits by branch, in the order each branch first
|
||||
// appears.
|
||||
func groupHits(hits bleveSearch.DocumentMatchCollection) ([]*groupEntries, error) {
|
||||
var groups []*groupEntries
|
||||
|
||||
byBranch := make(map[string]*groupEntries)
|
||||
|
||||
for _, hit := range hits {
|
||||
branch, _ := hit.Fields[_fieldBranchID].(string)
|
||||
|
||||
g, ok := byBranch[branch]
|
||||
if !ok {
|
||||
doc, _ := hit.Fields[_fieldDocumentID].(string)
|
||||
|
||||
documentID, err := xid.FromString(doc)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decoding document id of entry %s: %w", hit.ID, err)
|
||||
}
|
||||
|
||||
branchID, err := xid.FromString(branch)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decoding branch id of entry %s: %w", hit.ID, err)
|
||||
}
|
||||
|
||||
g = &groupEntries{
|
||||
documentID: documentID,
|
||||
branchID: branchID,
|
||||
}
|
||||
|
||||
byBranch[branch] = g
|
||||
groups = append(groups, g)
|
||||
}
|
||||
|
||||
if typ, _ := hit.Fields[_fieldType].(string); typ == "document" {
|
||||
g.titleID = hit.ID
|
||||
continue
|
||||
}
|
||||
|
||||
g.hitIDs = append(g.hitIDs, hit.ID)
|
||||
}
|
||||
|
||||
return groups, nil
|
||||
}
|
||||
561
server/core/internal/search/group_test.go
Normal file
561
server/core/internal/search/group_test.go
Normal file
|
|
@ -0,0 +1,561 @@
|
|||
package search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"log/slog"
|
||||
"testing"
|
||||
|
||||
bleveSearch "github.com/blevesearch/bleve/v2/search"
|
||||
"github.com/guregu/null/v5"
|
||||
"github.com/oxynote/oxynote/server/core/pkg/testutil"
|
||||
"github.com/rs/xid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func Test_GroupQuery_fingerprint(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
gq := GroupQuery{
|
||||
OrganizationID: "org-1",
|
||||
Query: "shipment",
|
||||
Limit: 5,
|
||||
PageToken: "token",
|
||||
}
|
||||
|
||||
pinned := gq
|
||||
pinned.CurrentDocumentID = null.ValueFrom(xid.New())
|
||||
|
||||
other := gq
|
||||
other.Query = "shipments"
|
||||
|
||||
// the organization, limit and token do not change the order a token
|
||||
// points into.
|
||||
same := gq
|
||||
same.OrganizationID = "org-2"
|
||||
same.Limit = 10
|
||||
same.PageToken = ""
|
||||
|
||||
assert.NotEmpty(t, gq.fingerprint())
|
||||
assert.Equal(t, gq.fingerprint(), same.fingerprint())
|
||||
assert.NotEqual(t, gq.fingerprint(), pinned.fingerprint())
|
||||
assert.NotEqual(t, gq.fingerprint(), other.fingerprint())
|
||||
}
|
||||
|
||||
func Test_GroupPage_BranchIDs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
b1, b2 := xid.New(), xid.New()
|
||||
|
||||
assert.Empty(t, GroupPage{}.BranchIDs())
|
||||
assert.Equal(t, []xid.ID{b1, b2}, GroupPage{
|
||||
Groups: []Group{
|
||||
{BranchID: b1},
|
||||
{BranchID: b2},
|
||||
},
|
||||
}.BranchIDs())
|
||||
}
|
||||
|
||||
func Test_GroupPage_DataSourceIDs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assert.Empty(t, GroupPage{}.DataSourceIDs())
|
||||
assert.Equal(t, []string{"ds-1", "ds-2"}, GroupPage{
|
||||
Groups: []Group{
|
||||
{Hits: []Block{{Attrs: map[string]string{"dataSourceId": "ds-1"}}, {}, {Attrs: map[string]string{"dataSourceId": "ds-2"}}}},
|
||||
{Hits: []Block{{Attrs: map[string]string{"dataSourceId": "ds-1"}}}},
|
||||
},
|
||||
}.DataSourceIDs())
|
||||
}
|
||||
|
||||
func Test_Index_SearchGroups(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
idx := openIndex(t, indexPath(t), &stubSource{})
|
||||
t.Cleanup(func() { require.NoError(t, idx.Close()) })
|
||||
|
||||
shipments := Scope{
|
||||
OrganizationID: "org-1",
|
||||
DocumentID: xid.New(),
|
||||
BranchID: xid.New(),
|
||||
BranchName: "main",
|
||||
BranchDefault: true,
|
||||
}
|
||||
|
||||
fork := shipments
|
||||
fork.BranchID = xid.New()
|
||||
fork.BranchName = "draft"
|
||||
|
||||
kafka := Scope{
|
||||
OrganizationID: "org-1",
|
||||
DocumentID: xid.New(),
|
||||
BranchID: xid.New(),
|
||||
BranchName: "main",
|
||||
BranchDefault: true,
|
||||
}
|
||||
|
||||
other := Scope{
|
||||
OrganizationID: "org-2",
|
||||
DocumentID: xid.New(),
|
||||
BranchID: xid.New(),
|
||||
BranchName: "main",
|
||||
BranchDefault: true,
|
||||
}
|
||||
|
||||
code := shipments.Block("c1", "codeBlock", "POST /v1/shipments")
|
||||
code.Attrs = map[string]string{"language": "http"}
|
||||
|
||||
chart := shipments.Block("m1", "metricBlock", "Shipments created per minute")
|
||||
chart.Attrs = map[string]string{
|
||||
"dataSourceId": "ds-1",
|
||||
"visualizationType": "timeseries",
|
||||
}
|
||||
|
||||
forkCode := code
|
||||
forkCode.ID = fork.BranchID.String() + "-c1"
|
||||
forkCode.BranchID = fork.BranchID
|
||||
forkCode.BranchName = fork.BranchName
|
||||
forkCode.BranchDefault = false
|
||||
|
||||
for branchID, entries := range map[xid.ID]map[string]Block{
|
||||
shipments.BranchID: {
|
||||
"docname": shipments.Block("docname", "document", "Shipments"),
|
||||
"h1": shipments.Block("h1", "heading", "Create a shipment"),
|
||||
"c1": code,
|
||||
"m1": chart,
|
||||
},
|
||||
fork.BranchID: {
|
||||
"c1": forkCode,
|
||||
},
|
||||
kafka.BranchID: {
|
||||
"docname": kafka.Block("docname", "document", "Kafka topics"),
|
||||
"p1": kafka.Block("p1", "paragraph", "shipment events"),
|
||||
"p2": kafka.Block("p2", "paragraph", "a11y & <tags>"),
|
||||
},
|
||||
other.BranchID: {
|
||||
"p1": other.Block("p1", "paragraph", "shipment"),
|
||||
},
|
||||
} {
|
||||
require.NoError(t, idx.ReplaceBranch(context.Background(), branchID, entries))
|
||||
}
|
||||
|
||||
// mark returns the block with its text replaced by the highlighted one.
|
||||
mark := func(b Block, text string) Block {
|
||||
b.Text = text
|
||||
|
||||
return b
|
||||
}
|
||||
|
||||
shipmentsGroup := Group{
|
||||
DocumentID: shipments.DocumentID,
|
||||
BranchID: shipments.BranchID,
|
||||
Title: null.StringFrom("<mark>Shipments</mark>"),
|
||||
Hits: []Block{
|
||||
mark(shipments.Block("h1", "heading", "Create a shipment"), "Create a <mark>shipment</mark>"),
|
||||
mark(code, "POST /v1/<mark>shipments</mark>"),
|
||||
mark(chart, "<mark>Shipments</mark> created per minute"),
|
||||
},
|
||||
TotalHits: 3,
|
||||
}
|
||||
|
||||
kafkaGroup := Group{
|
||||
DocumentID: kafka.DocumentID,
|
||||
BranchID: kafka.BranchID,
|
||||
Hits: []Block{
|
||||
mark(kafka.Block("p1", "paragraph", "shipment events"), "<mark>shipment</mark> events"),
|
||||
},
|
||||
TotalHits: 1,
|
||||
}
|
||||
|
||||
forkGroup := Group{
|
||||
DocumentID: shipments.DocumentID,
|
||||
BranchID: fork.BranchID,
|
||||
Hits: []Block{
|
||||
mark(forkCode, "POST /v1/<mark>shipments</mark>"),
|
||||
},
|
||||
TotalHits: 1,
|
||||
}
|
||||
|
||||
// shipmentsGroup cut to its first two hits.
|
||||
shipmentsCut := shipmentsGroup
|
||||
shipmentsCut.Hits = shipmentsGroup.Hits[:2]
|
||||
shipmentsCut.NextHitsToken = null.StringFrom(encodePageToken(2, BranchQuery{
|
||||
BranchID: shipments.BranchID,
|
||||
Query: "shipment",
|
||||
}.fingerprint()))
|
||||
|
||||
// token returns the page token of the offset for the query.
|
||||
token := func(gq GroupQuery, offset int) string {
|
||||
return encodePageToken(offset, gq.fingerprint())
|
||||
}
|
||||
|
||||
cancelled, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
cc := map[string]struct {
|
||||
Context context.Context
|
||||
Query GroupQuery
|
||||
Result GroupPage
|
||||
Err error
|
||||
}{
|
||||
"Invalid query": {
|
||||
Query: GroupQuery{OrganizationID: "org-1"},
|
||||
Err: ErrInvalidQuery,
|
||||
},
|
||||
"Invalid page token": {
|
||||
Query: GroupQuery{OrganizationID: "org-1", Query: "shipment", PageToken: "%%%"},
|
||||
Err: ErrInvalidPageToken,
|
||||
},
|
||||
"Page token of another query": {
|
||||
Query: GroupQuery{
|
||||
OrganizationID: "org-1",
|
||||
Query: "shipment",
|
||||
PageToken: token(GroupQuery{Query: "kafka"}, 1),
|
||||
},
|
||||
Err: ErrInvalidPageToken,
|
||||
},
|
||||
"Error returned by the index": {
|
||||
Context: cancelled,
|
||||
Query: GroupQuery{OrganizationID: "org-1", Query: "shipment"},
|
||||
Err: assert.AnError,
|
||||
},
|
||||
"No matches": {
|
||||
Query: GroupQuery{OrganizationID: "org-1", Query: "zebra"},
|
||||
Result: GroupPage{
|
||||
Groups: []Group{},
|
||||
},
|
||||
},
|
||||
"Successful search": {
|
||||
Query: GroupQuery{OrganizationID: "org-1", Query: "shipment"},
|
||||
Result: GroupPage{
|
||||
Groups: []Group{shipmentsGroup, kafkaGroup, forkGroup},
|
||||
TotalHits: 5,
|
||||
TotalDocuments: 3,
|
||||
},
|
||||
},
|
||||
"First page": {
|
||||
Query: GroupQuery{OrganizationID: "org-1", Query: "shipment", Limit: 2},
|
||||
Result: GroupPage{
|
||||
Groups: []Group{shipmentsGroup, kafkaGroup},
|
||||
TotalHits: 5,
|
||||
TotalDocuments: 3,
|
||||
NextPageToken: null.StringFrom(token(GroupQuery{Query: "shipment"}, 2)),
|
||||
},
|
||||
},
|
||||
"Last page": {
|
||||
Query: GroupQuery{
|
||||
OrganizationID: "org-1",
|
||||
Query: "shipment",
|
||||
Limit: 2,
|
||||
PageToken: token(GroupQuery{Query: "shipment"}, 2),
|
||||
},
|
||||
Result: GroupPage{
|
||||
Groups: []Group{forkGroup},
|
||||
TotalHits: 5,
|
||||
TotalDocuments: 3,
|
||||
},
|
||||
},
|
||||
"Page past the end": {
|
||||
Query: GroupQuery{
|
||||
OrganizationID: "org-1",
|
||||
Query: "shipment",
|
||||
PageToken: token(GroupQuery{Query: "shipment"}, 10),
|
||||
},
|
||||
Result: GroupPage{
|
||||
Groups: []Group{},
|
||||
TotalHits: 5,
|
||||
TotalDocuments: 3,
|
||||
},
|
||||
},
|
||||
"Current document first": {
|
||||
Query: GroupQuery{
|
||||
OrganizationID: "org-1",
|
||||
Query: "shipment",
|
||||
CurrentDocumentID: null.ValueFrom(kafka.DocumentID),
|
||||
},
|
||||
Result: GroupPage{
|
||||
Groups: []Group{kafkaGroup, shipmentsGroup, forkGroup},
|
||||
TotalHits: 5,
|
||||
TotalDocuments: 3,
|
||||
},
|
||||
},
|
||||
"Current document keeps its branches in match order": {
|
||||
Query: GroupQuery{
|
||||
OrganizationID: "org-1",
|
||||
Query: "shipment",
|
||||
CurrentDocumentID: null.ValueFrom(shipments.DocumentID),
|
||||
},
|
||||
Result: GroupPage{
|
||||
Groups: []Group{shipmentsGroup, forkGroup, kafkaGroup},
|
||||
TotalHits: 5,
|
||||
TotalDocuments: 3,
|
||||
},
|
||||
},
|
||||
"Hits over the group limit": {
|
||||
Query: GroupQuery{OrganizationID: "org-1", Query: "shipment", HitsLimit: 2},
|
||||
Result: GroupPage{
|
||||
Groups: []Group{shipmentsCut, kafkaGroup, forkGroup},
|
||||
TotalHits: 5,
|
||||
TotalDocuments: 3,
|
||||
},
|
||||
},
|
||||
"Title match without hits": {
|
||||
Query: GroupQuery{OrganizationID: "org-1", Query: "kafka"},
|
||||
Result: GroupPage{
|
||||
Groups: []Group{
|
||||
{
|
||||
DocumentID: kafka.DocumentID,
|
||||
BranchID: kafka.BranchID,
|
||||
Title: null.StringFrom("<mark>Kafka</mark> topics"),
|
||||
Hits: []Block{},
|
||||
},
|
||||
},
|
||||
TotalDocuments: 1,
|
||||
},
|
||||
},
|
||||
"Text is escaped around the marks": {
|
||||
Query: GroupQuery{OrganizationID: "org-1", Query: "accessibility"},
|
||||
Result: GroupPage{
|
||||
Groups: []Group{
|
||||
{
|
||||
DocumentID: kafka.DocumentID,
|
||||
BranchID: kafka.BranchID,
|
||||
Hits: []Block{
|
||||
mark(kafka.Block("p2", "paragraph", ""), "<mark>a11y</mark> & <tags>"),
|
||||
},
|
||||
TotalHits: 1,
|
||||
},
|
||||
},
|
||||
TotalHits: 1,
|
||||
TotalDocuments: 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for cn, c := range cc {
|
||||
t.Run(cn, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := c.Context
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
|
||||
res, err := idx.SearchGroups(ctx, c.Query)
|
||||
testutil.AssertEqualError(t, c.Err, err)
|
||||
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
assert.Equal(t, c.Result, res)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_GroupLimit(t *testing.T) {
|
||||
cc := map[string]struct {
|
||||
Limit int
|
||||
Result int
|
||||
}{
|
||||
"No limit": {Limit: 0, Result: GroupLimitDefault},
|
||||
"Negative limit": {Limit: -1, Result: GroupLimitDefault},
|
||||
"Limit within caps": {Limit: 5, Result: 5},
|
||||
"Limit over the cap": {
|
||||
Limit: GroupLimitMax + 1,
|
||||
Result: GroupLimitMax,
|
||||
},
|
||||
}
|
||||
|
||||
for cn, c := range cc {
|
||||
t.Run(cn, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assert.Equal(t, c.Result, GroupLimit(c.Limit))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_GroupHitsLimit(t *testing.T) {
|
||||
cc := map[string]struct {
|
||||
Limit int
|
||||
Result int
|
||||
}{
|
||||
"No limit": {Limit: 0, Result: GroupHitsLimitDefault},
|
||||
"Negative limit": {Limit: -1, Result: GroupHitsLimitDefault},
|
||||
"Limit within caps": {Limit: 5, Result: 5},
|
||||
"Limit over the cap": {
|
||||
Limit: GroupHitsLimitMax + 1,
|
||||
Result: GroupHitsLimitMax,
|
||||
},
|
||||
}
|
||||
|
||||
for cn, c := range cc {
|
||||
t.Run(cn, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assert.Equal(t, c.Result, GroupHitsLimit(c.Limit))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_Index_decodeHighlightedHit(t *testing.T) {
|
||||
documentID, branchID := xid.New(), xid.New()
|
||||
|
||||
fields := map[string]any{
|
||||
_fieldOrganizationID: "org-1",
|
||||
_fieldDocumentID: documentID.String(),
|
||||
_fieldBranchID: branchID.String(),
|
||||
_fieldBranchName: "main",
|
||||
_fieldBranchDefault: true,
|
||||
_fieldType: "paragraph",
|
||||
_fieldText: "a & b",
|
||||
}
|
||||
|
||||
block := Block{
|
||||
ID: "e1",
|
||||
OrganizationID: "org-1",
|
||||
DocumentID: documentID,
|
||||
BranchID: branchID,
|
||||
BranchName: "main",
|
||||
BranchDefault: true,
|
||||
Type: "paragraph",
|
||||
}
|
||||
|
||||
// withText returns the block with the text.
|
||||
withText := func(text string) Block {
|
||||
b := block
|
||||
b.Text = text
|
||||
|
||||
return b
|
||||
}
|
||||
|
||||
cc := map[string]struct {
|
||||
Hit *bleveSearch.DocumentMatch
|
||||
Result Block
|
||||
Err error
|
||||
}{
|
||||
"Error returned by decodeHit": {
|
||||
Hit: &bleveSearch.DocumentMatch{ID: "e1", Fields: map[string]any{}},
|
||||
Err: assert.AnError,
|
||||
},
|
||||
"Hit without a fragment is escaped": {
|
||||
Hit: &bleveSearch.DocumentMatch{ID: "e1", Fields: fields},
|
||||
Result: withText("a & b"),
|
||||
},
|
||||
"Hit with a fragment": {
|
||||
Hit: &bleveSearch.DocumentMatch{
|
||||
ID: "e1",
|
||||
Fields: fields,
|
||||
Fragments: bleveSearch.FieldFragmentMap{_fieldText: {"<mark>a</mark> & b"}},
|
||||
},
|
||||
Result: withText("<mark>a</mark> & b"),
|
||||
},
|
||||
}
|
||||
|
||||
for cn, c := range cc {
|
||||
t.Run(cn, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
idx := &Index{log: slog.New(slog.DiscardHandler)}
|
||||
|
||||
res, err := idx.decodeHighlightedHit(c.Hit)
|
||||
testutil.AssertEqualError(t, c.Err, err)
|
||||
assert.Equal(t, c.Result, res)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_encodePageToken(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
token := encodePageToken(30, "fp")
|
||||
|
||||
data, err := base64.RawURLEncoding.DecodeString(token)
|
||||
require.NoError(t, err)
|
||||
assert.JSONEq(t, `{"o":30,"f":"fp"}`, string(data))
|
||||
}
|
||||
|
||||
func Test_decodePageToken(t *testing.T) {
|
||||
encode := func(s string) string {
|
||||
return base64.RawURLEncoding.EncodeToString([]byte(s))
|
||||
}
|
||||
|
||||
cc := map[string]struct {
|
||||
Token string
|
||||
Result int
|
||||
Err error
|
||||
}{
|
||||
"Empty token": {Token: "", Result: 0},
|
||||
"Invalid base64": {Token: "%%%", Err: ErrInvalidPageToken},
|
||||
"Invalid JSON": {Token: encode("nope"), Err: ErrInvalidPageToken},
|
||||
"Negative offset": {Token: encode(`{"o":-1,"f":"fp"}`), Err: ErrInvalidPageToken},
|
||||
"Another fingerprint": {Token: encode(`{"o":1,"f":"other"}`), Err: ErrInvalidPageToken},
|
||||
"Valid token": {Token: encodePageToken(30, "fp"), Result: 30},
|
||||
}
|
||||
|
||||
for cn, c := range cc {
|
||||
t.Run(cn, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
res, err := decodePageToken(c.Token, "fp")
|
||||
testutil.AssertEqualError(t, c.Err, err)
|
||||
assert.Equal(t, c.Result, res)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_groupHits(t *testing.T) {
|
||||
documentID, branchID, forkID := xid.New(), xid.New(), xid.New()
|
||||
|
||||
// hit builds a match carrying the fields the grouping reads.
|
||||
hit := func(id, documentID, branchID, typ string) *bleveSearch.DocumentMatch {
|
||||
return &bleveSearch.DocumentMatch{
|
||||
ID: id,
|
||||
Fields: map[string]any{
|
||||
_fieldDocumentID: documentID,
|
||||
_fieldBranchID: branchID,
|
||||
_fieldType: typ,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
cc := map[string]struct {
|
||||
Hits bleveSearch.DocumentMatchCollection
|
||||
Result []*groupEntries
|
||||
Err error
|
||||
}{
|
||||
"Invalid document id": {
|
||||
Hits: bleveSearch.DocumentMatchCollection{hit("a", "nope", branchID.String(), "paragraph")},
|
||||
Err: assert.AnError,
|
||||
},
|
||||
"Invalid branch id": {
|
||||
Hits: bleveSearch.DocumentMatchCollection{hit("a", documentID.String(), "nope", "paragraph")},
|
||||
Err: assert.AnError,
|
||||
},
|
||||
"No hits": {},
|
||||
"Hits grouped by branch in first appearance order": {
|
||||
Hits: bleveSearch.DocumentMatchCollection{
|
||||
hit("a", documentID.String(), branchID.String(), "heading"),
|
||||
hit("b", documentID.String(), forkID.String(), "paragraph"),
|
||||
hit("c", documentID.String(), branchID.String(), "document"),
|
||||
hit("d", documentID.String(), branchID.String(), "paragraph"),
|
||||
},
|
||||
Result: []*groupEntries{
|
||||
{documentID: documentID, branchID: branchID, titleID: "c", hitIDs: []string{"a", "d"}},
|
||||
{documentID: documentID, branchID: forkID, hitIDs: []string{"b"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for cn, c := range cc {
|
||||
t.Run(cn, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
res, err := groupHits(c.Hits)
|
||||
testutil.AssertEqualError(t, c.Err, err)
|
||||
assert.Equal(t, c.Result, res)
|
||||
})
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user