diff --git a/datagen/go.mod b/datagen/go.mod index c948cb86..5a36408a 100644 --- a/datagen/go.mod +++ b/datagen/go.mod @@ -38,27 +38,25 @@ require ( github.com/jarcoal/httpmock v1.4.2 // indirect github.com/lib/pq v1.10.9 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/moby/api v1.55.0 // indirect - github.com/moby/moby/client v0.5.1 // indirect + github.com/moby/moby/api v1.56.0 // indirect + github.com/moby/moby/client v0.6.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/prometheus/common v0.71.0 // indirect + github.com/prometheus/common v0.72.0 // indirect github.com/prometheus/procfs v0.21.1 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/rs/xid v1.6.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 // indirect go.opentelemetry.io/otel v1.46.0 // indirect go.opentelemetry.io/otel/metric v1.46.0 // indirect - go.opentelemetry.io/otel/sdk v1.46.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.46.0 // indirect go.opentelemetry.io/otel/trace v1.46.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.28.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/sync v0.22.0 // indirect - golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.41.0 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect google.golang.org/protobuf v1.36.12 // indirect ) diff --git a/datagen/go.sum b/datagen/go.sum index 294e9315..0575242b 100644 --- a/datagen/go.sum +++ b/datagen/go.sum @@ -50,8 +50,8 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jarcoal/httpmock v1.4.2 h1:dKwiP/9zITCPfBLsDn3kchbSOu16JrnxtVEmL0fPRcI= github.com/jarcoal/httpmock v1.4.2/go.mod h1:ftW1xULwo+j0R0JJkJIIi7UKigZUXCLLanykgjwBXL0= -github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= -github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= @@ -60,10 +60,10 @@ github.com/maxatome/go-testdeep v1.14.0 h1:rRlLv1+kI8eOI3OaBXZwb3O7xY3exRzdW5QyX github.com/maxatome/go-testdeep v1.14.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc= -github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= -github.com/moby/moby/client v0.5.1 h1:tYNaJno4c0HXz12y5BiqEDy0rVTYkWzI26lGvnTMiJw= -github.com/moby/moby/client v0.5.1/go.mod h1:odLstlZ6uSnfvAgVxMpvgmb8SUdd+siH2T0GBuxVAlM= +github.com/moby/moby/api v1.56.0 h1:GQzua3NA599ASSIICx0iFgiJeO9YkdDARvQsm23ZZuQ= +github.com/moby/moby/api v1.56.0/go.mod h1:sZ+THbVWkjOmBPPfbnzdD/G1LuIexWhqlSHHPTDQ1Uk= +github.com/moby/moby/client v0.6.0 h1:AJjEB21QPbXSXjDsZorFBoDZPhMrfbpaPLgSMAW9Bgs= +github.com/moby/moby/client v0.6.0/go.mod h1:OCo00wNRyA3m4lmJ228W3JbyCN4ZNNYjpOXiJydBdcQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -81,8 +81,8 @@ github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59u github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo= github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM= -github.com/prometheus/common v0.71.0 h1:9KDAKb7Mj3HEVKyFCK6Dc/HIwlBzZIN2l7/lrHl3KK8= -github.com/prometheus/common v0.71.0/go.mod h1:CLJ5H8TEsGX8bl31BdMkfhIZ+QmZ9tBPPotUxUbfcmk= +github.com/prometheus/common v0.72.0 h1:tAYsE+sPJxIncDAobm4H5aQjmox9ZxEIIqPbiffa8G4= +github.com/prometheus/common v0.72.0/go.mod h1:77NWqAQ2tXT7BIK40qjJdw5Acrsrg1TlHAnsQi3i6mk= github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= @@ -96,8 +96,8 @@ github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWD github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU= go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= @@ -118,12 +118,12 @@ go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= -golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= -golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= -golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= -golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/server/core/internal/search/branch.go b/server/core/internal/search/branch.go new file mode 100644 index 00000000..36b9bac9 --- /dev/null +++ b/server/core/internal/search/branch.go @@ -0,0 +1,139 @@ +package search + +import ( + "context" + "fmt" + "hash/fnv" + "strconv" + + "github.com/blevesearch/bleve/v2" + bleveSearch "github.com/blevesearch/bleve/v2/search" + "github.com/guregu/null/v5" + "github.com/rs/xid" +) + +const ( + // BranchLimitDefault is the number of hits a branch page holds when + // the request names no limit. + BranchLimitDefault = 20 + + // BranchLimitMax caps the number of hits one branch page holds. + BranchLimitMax = 50 +) + +// BranchQuery describes one page of the hits of one branch. +type BranchQuery struct { + // OrganizationID specifies the organization searched. + OrganizationID string + + // BranchID specifies the branch searched. + BranchID xid.ID + + // Query specifies the text searched for. + Query string + + // Limit specifies the number of hits on the page, see BranchLimit. + Limit int + + // PageToken specifies the page, empty for the first one. A group's + // NextHitsToken continues after the hits the group shows. + PageToken string +} + +// fingerprint identifies the search a page token belongs to. A token +// replayed with another query or branch would point into another order. +func (bq BranchQuery) fingerprint() string { + h := fnv.New64a() + h.Write([]byte(bq.Query + "\x00" + bq.BranchID.String())) //nolint:gosec // hash writes never fail + + return strconv.FormatUint(h.Sum64(), 36) +} + +// BranchPage is one page of the hits of one branch. +type BranchPage struct { + // Hits specifies the page's matching content blocks, best first. + // Their text is HTML with the matched terms wrapped in mark tags. + Hits []Block + + // TotalHits specifies the number of matching content blocks across + // every page. + TotalHits int + + // NextPageToken specifies the token of the next page, if any. + NextPageToken null.String +} + +// SearchBranch searches one branch's content entries and returns one page +// of them, highlighted. The hits come in the order SearchGroups shows a +// group's hits in, so a group's NextHitsToken continues the group. +func (i *Index) SearchBranch(ctx context.Context, bq BranchQuery) (BranchPage, error) { + if err := ValidateQuery(bq.Query); err != nil { + return BranchPage{}, err + } + + fingerprint := bq.fingerprint() + + offset, err := decodePageToken(bq.PageToken, fingerprint) + if err != nil { + return BranchPage{}, err + } + + ctx, cancel := context.WithTimeout(ctx, _searchTimeout) + defer cancel() + + ctx = context.WithValue(ctx, bleveSearch.MaxTermSearchersKey, _maxTermSearchers) + + branch := bleve.NewTermQuery(bq.BranchID.String()) + branch.SetField(_fieldBranchID) + + // a document name is a group's title, never one of its hits. + name := bleve.NewTermQuery("document") + name.SetField(_fieldType) + + q := bleve.NewBooleanQuery() + q.AddMust(i.query(bq.OrganizationID, bq.Query), branch) + q.AddMustNot(name) + + limit := BranchLimit(bq.Limit) + + req := bleve.NewSearchRequestOptions(q, limit, offset, false) + req.Fields = []string{"*"} + req.SortBy([]string{"-_score", "_id"}) + req.Highlight = bleve.NewHighlightWithStyle(_highlighterName) + req.Highlight.AddField(_fieldText) + + res, err := i.idx.SearchInContext(ctx, req) + if err != nil { + return BranchPage{}, fmt.Errorf("searching branch: %w", err) + } + + page := BranchPage{ + Hits: make([]Block, 0, len(res.Hits)), + TotalHits: int(res.Total), + } + + for _, hit := range res.Hits { + b, err := i.decodeHighlightedHit(hit) + if err != nil { + return BranchPage{}, err + } + + page.Hits = append(page.Hits, b) + } + + if end := offset + len(res.Hits); end < page.TotalHits { + page.NextPageToken = null.StringFrom(encodePageToken(end, fingerprint)) + } + + return page, nil +} + +// BranchLimit returns the page size for the requested limit: the default +// when none is set, and at most BranchLimitMax. +func BranchLimit(limit int) int { + if limit <= 0 { + return BranchLimitDefault + } + + return min(limit, BranchLimitMax) +} diff --git a/server/core/internal/search/branch_test.go b/server/core/internal/search/branch_test.go new file mode 100644 index 00000000..6ff26dc4 --- /dev/null +++ b/server/core/internal/search/branch_test.go @@ -0,0 +1,250 @@ +package search + +import ( + "context" + "testing" + + "github.com/guregu/null/v5" + "github.com/oxynote/oxynote/server/core/pkg/testutil" + "github.com/rs/xid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_BranchQuery_fingerprint(t *testing.T) { + t.Parallel() + + bq := BranchQuery{ + OrganizationID: "org-1", + BranchID: xid.New(), + Query: "shipment", + Limit: 5, + PageToken: "token", + } + + branch := bq + branch.BranchID = xid.New() + + other := bq + other.Query = "shipments" + + // the organization, limit and token do not change the order a token + // points into. + same := bq + same.OrganizationID = "org-2" + same.Limit = 10 + same.PageToken = "" + + assert.NotEmpty(t, bq.fingerprint()) + assert.Equal(t, bq.fingerprint(), same.fingerprint()) + assert.NotEqual(t, bq.fingerprint(), branch.fingerprint()) + assert.NotEqual(t, bq.fingerprint(), other.fingerprint()) +} + +func Test_Index_SearchBranch(t *testing.T) { + t.Parallel() + + idx := openIndex(t, indexPath(t), &stubSource{}) + t.Cleanup(func() { require.NoError(t, idx.Close()) }) + + main := Scope{ + OrganizationID: "org-1", + DocumentID: xid.New(), + BranchID: xid.New(), + BranchName: "main", + BranchDefault: true, + } + + fork := main + fork.BranchID = xid.New() + fork.BranchName = "draft" + fork.BranchDefault = false + + other := Scope{ + OrganizationID: "org-2", + DocumentID: xid.New(), + BranchID: xid.New(), + BranchName: "main", + BranchDefault: true, + } + + for branchID, entries := range map[xid.ID]map[string]Block{ + main.BranchID: { + "docname": main.Block("docname", "document", "Shipments"), + "h1": main.Block("h1", "heading", "Shipment tracking"), + "p1": main.Block("p1", "paragraph", "a shipment is created"), + "p2": main.Block("p2", "paragraph", "every shipment has a label & a barcode"), + "p3": main.Block("p3", "paragraph", "each shipment arrives with a shipment note"), + "p4": main.Block("p4", "paragraph", "unrelated text"), + }, + fork.BranchID: { + "p1": fork.Block("p1", "paragraph", "a shipment is created"), + }, + other.BranchID: { + "p1": other.Block("p1", "paragraph", "shipment"), + }, + } { + require.NoError(t, idx.ReplaceBranch(context.Background(), branchID, entries)) + } + + query := BranchQuery{ + OrganizationID: "org-1", + BranchID: main.BranchID, + Query: "shipment", + } + + all, err := idx.SearchBranch(context.Background(), query) + require.NoError(t, err) + require.Len(t, all.Hits, 4) + + // with returns the query with the limit and the page token of the + // offset. + with := func(limit, offset int) BranchQuery { + bq := query + bq.Limit = limit + + if offset != 0 { + bq.PageToken = encodePageToken(offset, query.fingerprint()) + } + + return bq + } + + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + + type tcase struct { + Context context.Context + Query BranchQuery + Result BranchPage + Err error + } + + cc := map[string]tcase{ + "Invalid query": { + Query: BranchQuery{OrganizationID: "org-1", BranchID: main.BranchID}, + Err: ErrInvalidQuery, + }, + "Invalid page token": { + Query: BranchQuery{OrganizationID: "org-1", BranchID: main.BranchID, Query: "shipment", PageToken: "%%%"}, + Err: ErrInvalidPageToken, + }, + "Page token of another branch": func() tcase { + bq := query + bq.PageToken = encodePageToken(1, BranchQuery{BranchID: fork.BranchID, Query: "shipment"}.fingerprint()) + + return tcase{Query: bq, Err: ErrInvalidPageToken} + }(), + "Error returned by the index": { + Context: cancelled, + Query: query, + Err: assert.AnError, + }, + "No matches": func() tcase { + bq := query + bq.Query = "zebra" + + return tcase{Query: bq, Result: BranchPage{Hits: []Block{}}} + }(), + "Entries of another organization are never returned": func() tcase { + bq := query + bq.BranchID = other.BranchID + + return tcase{Query: bq, Result: BranchPage{Hits: []Block{}}} + }(), + "First page": { + Query: with(2, 0), + Result: BranchPage{ + Hits: all.Hits[:2], + TotalHits: 4, + NextPageToken: null.StringFrom(encodePageToken(2, query.fingerprint())), + }, + }, + "Last page": { + Query: with(2, 2), + Result: BranchPage{ + Hits: all.Hits[2:], + TotalHits: 4, + }, + }, + "Page past the end": { + Query: with(2, 10), + Result: BranchPage{ + Hits: []Block{}, + TotalHits: 4, + }, + }, + } + + for cn, c := range cc { + t.Run(cn, func(t *testing.T) { + t.Parallel() + + ctx := c.Context + if ctx == nil { + ctx = context.Background() + } + + res, err := idx.SearchBranch(ctx, c.Query) + testutil.AssertEqualError(t, c.Err, err) + + if err != nil { + return + } + + assert.Equal(t, c.Result, res) + }) + } + + t.Run("Continues the hits of a group", func(t *testing.T) { + t.Parallel() + + gp, err := idx.SearchGroups(context.Background(), GroupQuery{ + OrganizationID: "org-1", + Query: "shipment", + HitsLimit: 2, + }) + require.NoError(t, err) + + var group Group + + for _, g := range gp.Groups { + if g.BranchID == main.BranchID { + group = g + } + } + + require.True(t, group.NextHitsToken.Valid) + + bq := query + bq.PageToken = group.NextHitsToken.String + + res, err := idx.SearchBranch(context.Background(), bq) + require.NoError(t, err) + + assert.Equal(t, all.Hits, append(group.Hits, res.Hits...)) + }) +} + +func Test_BranchLimit(t *testing.T) { + cc := map[string]struct { + Limit int + Result int + }{ + "No limit": {Limit: 0, Result: BranchLimitDefault}, + "Negative limit": {Limit: -1, Result: BranchLimitDefault}, + "Limit within caps": {Limit: 5, Result: 5}, + "Limit over the cap": { + Limit: BranchLimitMax + 1, + Result: BranchLimitMax, + }, + } + + for cn, c := range cc { + t.Run(cn, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, c.Result, BranchLimit(c.Limit)) + }) + } +} diff --git a/server/core/internal/search/group.go b/server/core/internal/search/group.go new file mode 100644 index 00000000..1a2f10f0 --- /dev/null +++ b/server/core/internal/search/group.go @@ -0,0 +1,461 @@ +package search + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "hash/fnv" + "html" + "net/http" + "slices" + "strconv" + + "github.com/blevesearch/bleve/v2" + bleveSearch "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/query" + "github.com/guregu/null/v5" + "github.com/oxynote/oxynote/server/core/internal/document" + "github.com/oxynote/oxynote/server/core/pkg/errutil" + "github.com/rs/xid" +) + +const ( + // GroupLimitDefault is the number of groups a page holds when the + // request names no limit. + GroupLimitDefault = 30 + + // GroupLimitMax caps the number of groups one page holds. + GroupLimitMax = 50 + + // GroupHitsLimitDefault is the number of hits a group shows when the + // request names no limit. + GroupHitsLimitDefault = 3 + + // GroupHitsLimitMax caps the number of hits one group shows. + GroupHitsLimitMax = 10 + + // _groupHitCap caps the hits one grouped search reads. The groups and + // totals are built from these hits only. + _groupHitCap = 1000 +) + +// ErrInvalidPageToken is returned when a page token is malformed or was +// issued for another search. +var ErrInvalidPageToken = errutil.New(http.StatusBadRequest, "document.invalid_search_page_token", "invalid search page token") + +// GroupQuery describes one page of a grouped search. +type GroupQuery struct { + // OrganizationID specifies the organization searched. + OrganizationID string + + // Query specifies the text searched for. + Query string + + // CurrentDocumentID specifies the document whose groups come first. + CurrentDocumentID null.Value[xid.ID] + + // Limit specifies the number of groups on the page, see GroupLimit. + Limit int + + // HitsLimit specifies the number of hits each group shows, see + // GroupHitsLimit. + HitsLimit int + + // PageToken specifies the page, empty for the first one. + PageToken string +} + +// fingerprint identifies the search a page token belongs to. A token +// replayed with another query or pin would point into another order. +func (gq GroupQuery) fingerprint() string { + h := fnv.New64a() + h.Write([]byte(gq.Query + "\x00" + gq.CurrentDocumentID.V.String())) //nolint:gosec // hash writes never fail + + return strconv.FormatUint(h.Sum64(), 36) +} + +// Group is one branch of a document holding matches. +type Group struct { + // DocumentID specifies the document the branch belongs to. + DocumentID xid.ID + + // BranchID specifies the branch. + BranchID xid.ID + + // Title specifies the highlighted document name, set only when the + // name matched. + Title null.String + + // Hits specifies the first matching content blocks, best first. Their + // text is HTML with the matched terms wrapped in mark tags. + Hits []Block + + // TotalHits specifies the number of matching content blocks in the + // branch, counted like GroupPage.TotalHits. + TotalHits int + + // NextHitsToken specifies the SearchBranch page token of the hits + // after Hits, if any. + NextHitsToken null.String +} + +// GroupPage is one page of a grouped search. +type GroupPage struct { + // Groups specifies the page's groups, best first. + Groups []Group + + // TotalHits specifies the number of matching content blocks across + // every page. + TotalHits int + + // TotalDocuments specifies the number of groups across every page. + TotalDocuments int + + // Capped indicates that more matches exist than the totals count. + Capped bool + + // NextPageToken specifies the token of the next page, if any. + NextPageToken null.String +} + +// BranchIDs returns the ids of the page's branches. +func (gp GroupPage) BranchIDs() []xid.ID { + ids := make([]xid.ID, 0, len(gp.Groups)) + + for _, g := range gp.Groups { + ids = append(ids, g.BranchID) + } + + return ids +} + +// DataSourceIDs returns the ids of the data sources the page's hits +// query, without repeats. +func (gp GroupPage) DataSourceIDs() []string { + var ids []string + + for _, g := range gp.Groups { + for _, h := range g.Hits { + id := h.Attrs[document.AttrDataSourceID] + if id != "" && !slices.Contains(ids, id) { + ids = append(ids, id) + } + } + } + + return ids +} + +// pageToken is the decoded form of a page token. +type pageToken struct { + // Offset specifies the index of the page's first group or hit. + Offset int `json:"o"` + + // Fingerprint specifies the search the token belongs to. + Fingerprint string `json:"f"` +} + +// groupEntries collects the ids of one group's matching entries. +type groupEntries struct { + // documentID specifies the document the branch belongs to. + documentID xid.ID + + // branchID specifies the branch. + branchID xid.ID + + // titleID specifies the id of the matching document name entry, if + // any. + titleID string + + // hitIDs specifies the ids of the matching content entries, best + // first. + hitIDs []string +} + +// SearchGroups searches the organization's entries and returns one page +// of them grouped by branch. Groups are ordered by their best match, with +// the current document's groups first. A matching document name becomes +// the group's title instead of a hit. Each group shows its first hits +// only; SearchBranch pages through the rest. +// +// Grouping needs every match, which bleve cannot do in the index, so +// the first _groupHitCap matches are read and grouped here on every +// request. A page token is an offset into that order. Only the page's +// own entries are then read in full and highlighted. +func (i *Index) SearchGroups(ctx context.Context, gq GroupQuery) (GroupPage, error) { + if err := ValidateQuery(gq.Query); err != nil { + return GroupPage{}, err + } + + fingerprint := gq.fingerprint() + + offset, err := decodePageToken(gq.PageToken, fingerprint) + if err != nil { + return GroupPage{}, err + } + + ctx, cancel := context.WithTimeout(ctx, _searchTimeout) + defer cancel() + + ctx = context.WithValue(ctx, bleveSearch.MaxTermSearchersKey, _maxTermSearchers) + + q := i.query(gq.OrganizationID, gq.Query) + + req := bleve.NewSearchRequestOptions(q, _groupHitCap, 0, false) + req.Fields = []string{_fieldDocumentID, _fieldBranchID, _fieldType} + req.SortBy([]string{"-_score", "_id"}) + + res, err := i.idx.SearchInContext(ctx, req) + if err != nil { + return GroupPage{}, fmt.Errorf("searching documents: %w", err) + } + + groups, err := groupHits(res.Hits) + if err != nil { + return GroupPage{}, err + } + + if gq.CurrentDocumentID.Valid { + // a stable sort keeps both parts in their match order. + slices.SortStableFunc(groups, func(a, b *groupEntries) int { + ac := a.documentID == gq.CurrentDocumentID.V + bc := b.documentID == gq.CurrentDocumentID.V + + switch { + case ac && !bc: + return -1 + case bc && !ac: + return 1 + default: + return 0 + } + }) + } + + page := GroupPage{ + TotalDocuments: len(groups), + Capped: res.Total > uint64(len(res.Hits)), + } + + for _, g := range groups { + page.TotalHits += len(g.hitIDs) + } + + limit := GroupLimit(gq.Limit) + start := min(offset, len(groups)) + end := min(start+limit, len(groups)) + + if end < len(groups) { + page.NextPageToken = null.StringFrom(encodePageToken(end, fingerprint)) + } + + page.Groups, err = i.highlightGroups(ctx, gq, q, groups[start:end]) + if err != nil { + return GroupPage{}, err + } + + return page, nil +} + +// highlightGroups reads the groups' title and first hit entries in full, +// highlighted against the query. An entry removed since the groups were +// built is left out. +func (i *Index) highlightGroups(ctx context.Context, gq GroupQuery, q query.Query, groups []*groupEntries) ([]Group, error) { + res := make([]Group, 0, len(groups)) + hitsLimit := GroupHitsLimit(gq.HitsLimit) + + var ids []string + + for _, g := range groups { + if g.titleID != "" { + ids = append(ids, g.titleID) + } + + ids = append(ids, g.hitIDs[:min(hitsLimit, len(g.hitIDs))]...) + } + + if len(ids) == 0 { + return res, nil + } + + req := bleve.NewSearchRequestOptions( + bleve.NewConjunctionQuery(q, bleve.NewDocIDQuery(ids)), + len(ids), + 0, + false, + ) + req.Fields = []string{"*"} + req.Highlight = bleve.NewHighlightWithStyle(_highlighterName) + req.Highlight.AddField(_fieldText) + + sr, err := i.idx.SearchInContext(ctx, req) + if err != nil { + return nil, fmt.Errorf("highlighting documents: %w", err) + } + + blocks := make(map[string]Block, len(sr.Hits)) + + for _, hit := range sr.Hits { + b, err := i.decodeHighlightedHit(hit) + if err != nil { + return nil, err + } + + blocks[hit.ID] = b + } + + for _, g := range groups { + grp := Group{ + DocumentID: g.documentID, + BranchID: g.branchID, + Hits: []Block{}, + TotalHits: len(g.hitIDs), + } + + if b, ok := blocks[g.titleID]; ok { + grp.Title = null.StringFrom(b.Text) + } + + for _, id := range g.hitIDs[:min(hitsLimit, len(g.hitIDs))] { + if b, ok := blocks[id]; ok { + grp.Hits = append(grp.Hits, b) + } + } + + if len(g.hitIDs) > hitsLimit { + bq := BranchQuery{ + BranchID: g.branchID, + Query: gq.Query, + } + + grp.NextHitsToken = null.StringFrom(encodePageToken(hitsLimit, bq.fingerprint())) + } + + res = append(res, grp) + } + + return res, nil +} + +// GroupLimit returns the page size for the requested limit: the default +// when none is set, and at most GroupLimitMax. +func GroupLimit(limit int) int { + if limit <= 0 { + return GroupLimitDefault + } + + return min(limit, GroupLimitMax) +} + +// GroupHitsLimit returns the number of hits a group shows for the +// requested limit: the default when none is set, and at most +// GroupHitsLimitMax. +func GroupHitsLimit(limit int) int { + if limit <= 0 { + return GroupHitsLimitDefault + } + + return min(limit, GroupHitsLimitMax) +} + +// decodeHighlightedHit rebuilds an entry from a highlighted hit. Its text +// is the best fragment, or the full text escaped like a fragment is when +// the hit has none. +func (i *Index) decodeHighlightedHit(hit *bleveSearch.DocumentMatch) (Block, error) { + b, err := i.decodeHit(hit.ID, hit.Fields) + if err != nil { + return Block{}, err + } + + b.Text = html.EscapeString(b.Text) + + if fragments := hit.Fragments[_fieldText]; len(fragments) != 0 { + b.Text = fragments[0] + } + + return b, nil +} + +// encodePageToken encodes the offset of a page of the search. +func encodePageToken(offset int, fingerprint string) string { + data, err := json.Marshal(pageToken{ + Offset: offset, + Fingerprint: fingerprint, + }) + if err != nil { + // NOCOV: a struct of an int and a string always marshals. + return "" + } + + return base64.RawURLEncoding.EncodeToString(data) +} + +// decodePageToken returns the offset a page token points at. An empty +// token is the first page. +func decodePageToken(token, fingerprint string) (int, error) { + if token == "" { + return 0, nil + } + + data, err := base64.RawURLEncoding.DecodeString(token) + if err != nil { + return 0, ErrInvalidPageToken + } + + var pt pageToken + + if err := json.Unmarshal(data, &pt); err != nil { + return 0, ErrInvalidPageToken + } + + if pt.Offset < 0 || pt.Fingerprint != fingerprint { + return 0, ErrInvalidPageToken + } + + return pt.Offset, nil +} + +// groupHits groups the hits by branch, in the order each branch first +// appears. +func groupHits(hits bleveSearch.DocumentMatchCollection) ([]*groupEntries, error) { + var groups []*groupEntries + + byBranch := make(map[string]*groupEntries) + + for _, hit := range hits { + branch, _ := hit.Fields[_fieldBranchID].(string) + + g, ok := byBranch[branch] + if !ok { + doc, _ := hit.Fields[_fieldDocumentID].(string) + + documentID, err := xid.FromString(doc) + if err != nil { + return nil, fmt.Errorf("decoding document id of entry %s: %w", hit.ID, err) + } + + branchID, err := xid.FromString(branch) + if err != nil { + return nil, fmt.Errorf("decoding branch id of entry %s: %w", hit.ID, err) + } + + g = &groupEntries{ + documentID: documentID, + branchID: branchID, + } + + byBranch[branch] = g + groups = append(groups, g) + } + + if typ, _ := hit.Fields[_fieldType].(string); typ == "document" { + g.titleID = hit.ID + continue + } + + g.hitIDs = append(g.hitIDs, hit.ID) + } + + return groups, nil +} diff --git a/server/core/internal/search/group_test.go b/server/core/internal/search/group_test.go new file mode 100644 index 00000000..418e6ec1 --- /dev/null +++ b/server/core/internal/search/group_test.go @@ -0,0 +1,561 @@ +package search + +import ( + "context" + "encoding/base64" + "log/slog" + "testing" + + bleveSearch "github.com/blevesearch/bleve/v2/search" + "github.com/guregu/null/v5" + "github.com/oxynote/oxynote/server/core/pkg/testutil" + "github.com/rs/xid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_GroupQuery_fingerprint(t *testing.T) { + t.Parallel() + + gq := GroupQuery{ + OrganizationID: "org-1", + Query: "shipment", + Limit: 5, + PageToken: "token", + } + + pinned := gq + pinned.CurrentDocumentID = null.ValueFrom(xid.New()) + + other := gq + other.Query = "shipments" + + // the organization, limit and token do not change the order a token + // points into. + same := gq + same.OrganizationID = "org-2" + same.Limit = 10 + same.PageToken = "" + + assert.NotEmpty(t, gq.fingerprint()) + assert.Equal(t, gq.fingerprint(), same.fingerprint()) + assert.NotEqual(t, gq.fingerprint(), pinned.fingerprint()) + assert.NotEqual(t, gq.fingerprint(), other.fingerprint()) +} + +func Test_GroupPage_BranchIDs(t *testing.T) { + t.Parallel() + + b1, b2 := xid.New(), xid.New() + + assert.Empty(t, GroupPage{}.BranchIDs()) + assert.Equal(t, []xid.ID{b1, b2}, GroupPage{ + Groups: []Group{ + {BranchID: b1}, + {BranchID: b2}, + }, + }.BranchIDs()) +} + +func Test_GroupPage_DataSourceIDs(t *testing.T) { + t.Parallel() + + assert.Empty(t, GroupPage{}.DataSourceIDs()) + assert.Equal(t, []string{"ds-1", "ds-2"}, GroupPage{ + Groups: []Group{ + {Hits: []Block{{Attrs: map[string]string{"dataSourceId": "ds-1"}}, {}, {Attrs: map[string]string{"dataSourceId": "ds-2"}}}}, + {Hits: []Block{{Attrs: map[string]string{"dataSourceId": "ds-1"}}}}, + }, + }.DataSourceIDs()) +} + +func Test_Index_SearchGroups(t *testing.T) { + t.Parallel() + + idx := openIndex(t, indexPath(t), &stubSource{}) + t.Cleanup(func() { require.NoError(t, idx.Close()) }) + + shipments := Scope{ + OrganizationID: "org-1", + DocumentID: xid.New(), + BranchID: xid.New(), + BranchName: "main", + BranchDefault: true, + } + + fork := shipments + fork.BranchID = xid.New() + fork.BranchName = "draft" + + kafka := Scope{ + OrganizationID: "org-1", + DocumentID: xid.New(), + BranchID: xid.New(), + BranchName: "main", + BranchDefault: true, + } + + other := Scope{ + OrganizationID: "org-2", + DocumentID: xid.New(), + BranchID: xid.New(), + BranchName: "main", + BranchDefault: true, + } + + code := shipments.Block("c1", "codeBlock", "POST /v1/shipments") + code.Attrs = map[string]string{"language": "http"} + + chart := shipments.Block("m1", "metricBlock", "Shipments created per minute") + chart.Attrs = map[string]string{ + "dataSourceId": "ds-1", + "visualizationType": "timeseries", + } + + forkCode := code + forkCode.ID = fork.BranchID.String() + "-c1" + forkCode.BranchID = fork.BranchID + forkCode.BranchName = fork.BranchName + forkCode.BranchDefault = false + + for branchID, entries := range map[xid.ID]map[string]Block{ + shipments.BranchID: { + "docname": shipments.Block("docname", "document", "Shipments"), + "h1": shipments.Block("h1", "heading", "Create a shipment"), + "c1": code, + "m1": chart, + }, + fork.BranchID: { + "c1": forkCode, + }, + kafka.BranchID: { + "docname": kafka.Block("docname", "document", "Kafka topics"), + "p1": kafka.Block("p1", "paragraph", "shipment events"), + "p2": kafka.Block("p2", "paragraph", "a11y & "), + }, + other.BranchID: { + "p1": other.Block("p1", "paragraph", "shipment"), + }, + } { + require.NoError(t, idx.ReplaceBranch(context.Background(), branchID, entries)) + } + + // mark returns the block with its text replaced by the highlighted one. + mark := func(b Block, text string) Block { + b.Text = text + + return b + } + + shipmentsGroup := Group{ + DocumentID: shipments.DocumentID, + BranchID: shipments.BranchID, + Title: null.StringFrom("Shipments"), + Hits: []Block{ + mark(shipments.Block("h1", "heading", "Create a shipment"), "Create a shipment"), + mark(code, "POST /v1/shipments"), + mark(chart, "Shipments created per minute"), + }, + TotalHits: 3, + } + + kafkaGroup := Group{ + DocumentID: kafka.DocumentID, + BranchID: kafka.BranchID, + Hits: []Block{ + mark(kafka.Block("p1", "paragraph", "shipment events"), "shipment events"), + }, + TotalHits: 1, + } + + forkGroup := Group{ + DocumentID: shipments.DocumentID, + BranchID: fork.BranchID, + Hits: []Block{ + mark(forkCode, "POST /v1/shipments"), + }, + TotalHits: 1, + } + + // shipmentsGroup cut to its first two hits. + shipmentsCut := shipmentsGroup + shipmentsCut.Hits = shipmentsGroup.Hits[:2] + shipmentsCut.NextHitsToken = null.StringFrom(encodePageToken(2, BranchQuery{ + BranchID: shipments.BranchID, + Query: "shipment", + }.fingerprint())) + + // token returns the page token of the offset for the query. + token := func(gq GroupQuery, offset int) string { + return encodePageToken(offset, gq.fingerprint()) + } + + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + + cc := map[string]struct { + Context context.Context + Query GroupQuery + Result GroupPage + Err error + }{ + "Invalid query": { + Query: GroupQuery{OrganizationID: "org-1"}, + Err: ErrInvalidQuery, + }, + "Invalid page token": { + Query: GroupQuery{OrganizationID: "org-1", Query: "shipment", PageToken: "%%%"}, + Err: ErrInvalidPageToken, + }, + "Page token of another query": { + Query: GroupQuery{ + OrganizationID: "org-1", + Query: "shipment", + PageToken: token(GroupQuery{Query: "kafka"}, 1), + }, + Err: ErrInvalidPageToken, + }, + "Error returned by the index": { + Context: cancelled, + Query: GroupQuery{OrganizationID: "org-1", Query: "shipment"}, + Err: assert.AnError, + }, + "No matches": { + Query: GroupQuery{OrganizationID: "org-1", Query: "zebra"}, + Result: GroupPage{ + Groups: []Group{}, + }, + }, + "Successful search": { + Query: GroupQuery{OrganizationID: "org-1", Query: "shipment"}, + Result: GroupPage{ + Groups: []Group{shipmentsGroup, kafkaGroup, forkGroup}, + TotalHits: 5, + TotalDocuments: 3, + }, + }, + "First page": { + Query: GroupQuery{OrganizationID: "org-1", Query: "shipment", Limit: 2}, + Result: GroupPage{ + Groups: []Group{shipmentsGroup, kafkaGroup}, + TotalHits: 5, + TotalDocuments: 3, + NextPageToken: null.StringFrom(token(GroupQuery{Query: "shipment"}, 2)), + }, + }, + "Last page": { + Query: GroupQuery{ + OrganizationID: "org-1", + Query: "shipment", + Limit: 2, + PageToken: token(GroupQuery{Query: "shipment"}, 2), + }, + Result: GroupPage{ + Groups: []Group{forkGroup}, + TotalHits: 5, + TotalDocuments: 3, + }, + }, + "Page past the end": { + Query: GroupQuery{ + OrganizationID: "org-1", + Query: "shipment", + PageToken: token(GroupQuery{Query: "shipment"}, 10), + }, + Result: GroupPage{ + Groups: []Group{}, + TotalHits: 5, + TotalDocuments: 3, + }, + }, + "Current document first": { + Query: GroupQuery{ + OrganizationID: "org-1", + Query: "shipment", + CurrentDocumentID: null.ValueFrom(kafka.DocumentID), + }, + Result: GroupPage{ + Groups: []Group{kafkaGroup, shipmentsGroup, forkGroup}, + TotalHits: 5, + TotalDocuments: 3, + }, + }, + "Current document keeps its branches in match order": { + Query: GroupQuery{ + OrganizationID: "org-1", + Query: "shipment", + CurrentDocumentID: null.ValueFrom(shipments.DocumentID), + }, + Result: GroupPage{ + Groups: []Group{shipmentsGroup, forkGroup, kafkaGroup}, + TotalHits: 5, + TotalDocuments: 3, + }, + }, + "Hits over the group limit": { + Query: GroupQuery{OrganizationID: "org-1", Query: "shipment", HitsLimit: 2}, + Result: GroupPage{ + Groups: []Group{shipmentsCut, kafkaGroup, forkGroup}, + TotalHits: 5, + TotalDocuments: 3, + }, + }, + "Title match without hits": { + Query: GroupQuery{OrganizationID: "org-1", Query: "kafka"}, + Result: GroupPage{ + Groups: []Group{ + { + DocumentID: kafka.DocumentID, + BranchID: kafka.BranchID, + Title: null.StringFrom("Kafka topics"), + Hits: []Block{}, + }, + }, + TotalDocuments: 1, + }, + }, + "Text is escaped around the marks": { + Query: GroupQuery{OrganizationID: "org-1", Query: "accessibility"}, + Result: GroupPage{ + Groups: []Group{ + { + DocumentID: kafka.DocumentID, + BranchID: kafka.BranchID, + Hits: []Block{ + mark(kafka.Block("p2", "paragraph", ""), "a11y & <tags>"), + }, + TotalHits: 1, + }, + }, + TotalHits: 1, + TotalDocuments: 1, + }, + }, + } + + for cn, c := range cc { + t.Run(cn, func(t *testing.T) { + t.Parallel() + + ctx := c.Context + if ctx == nil { + ctx = context.Background() + } + + res, err := idx.SearchGroups(ctx, c.Query) + testutil.AssertEqualError(t, c.Err, err) + + if err != nil { + return + } + + assert.Equal(t, c.Result, res) + }) + } +} + +func Test_GroupLimit(t *testing.T) { + cc := map[string]struct { + Limit int + Result int + }{ + "No limit": {Limit: 0, Result: GroupLimitDefault}, + "Negative limit": {Limit: -1, Result: GroupLimitDefault}, + "Limit within caps": {Limit: 5, Result: 5}, + "Limit over the cap": { + Limit: GroupLimitMax + 1, + Result: GroupLimitMax, + }, + } + + for cn, c := range cc { + t.Run(cn, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, c.Result, GroupLimit(c.Limit)) + }) + } +} + +func Test_GroupHitsLimit(t *testing.T) { + cc := map[string]struct { + Limit int + Result int + }{ + "No limit": {Limit: 0, Result: GroupHitsLimitDefault}, + "Negative limit": {Limit: -1, Result: GroupHitsLimitDefault}, + "Limit within caps": {Limit: 5, Result: 5}, + "Limit over the cap": { + Limit: GroupHitsLimitMax + 1, + Result: GroupHitsLimitMax, + }, + } + + for cn, c := range cc { + t.Run(cn, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, c.Result, GroupHitsLimit(c.Limit)) + }) + } +} + +func Test_Index_decodeHighlightedHit(t *testing.T) { + documentID, branchID := xid.New(), xid.New() + + fields := map[string]any{ + _fieldOrganizationID: "org-1", + _fieldDocumentID: documentID.String(), + _fieldBranchID: branchID.String(), + _fieldBranchName: "main", + _fieldBranchDefault: true, + _fieldType: "paragraph", + _fieldText: "a & b", + } + + block := Block{ + ID: "e1", + OrganizationID: "org-1", + DocumentID: documentID, + BranchID: branchID, + BranchName: "main", + BranchDefault: true, + Type: "paragraph", + } + + // withText returns the block with the text. + withText := func(text string) Block { + b := block + b.Text = text + + return b + } + + cc := map[string]struct { + Hit *bleveSearch.DocumentMatch + Result Block + Err error + }{ + "Error returned by decodeHit": { + Hit: &bleveSearch.DocumentMatch{ID: "e1", Fields: map[string]any{}}, + Err: assert.AnError, + }, + "Hit without a fragment is escaped": { + Hit: &bleveSearch.DocumentMatch{ID: "e1", Fields: fields}, + Result: withText("a & b"), + }, + "Hit with a fragment": { + Hit: &bleveSearch.DocumentMatch{ + ID: "e1", + Fields: fields, + Fragments: bleveSearch.FieldFragmentMap{_fieldText: {"a & b"}}, + }, + Result: withText("a & b"), + }, + } + + for cn, c := range cc { + t.Run(cn, func(t *testing.T) { + t.Parallel() + + idx := &Index{log: slog.New(slog.DiscardHandler)} + + res, err := idx.decodeHighlightedHit(c.Hit) + testutil.AssertEqualError(t, c.Err, err) + assert.Equal(t, c.Result, res) + }) + } +} + +func Test_encodePageToken(t *testing.T) { + t.Parallel() + + token := encodePageToken(30, "fp") + + data, err := base64.RawURLEncoding.DecodeString(token) + require.NoError(t, err) + assert.JSONEq(t, `{"o":30,"f":"fp"}`, string(data)) +} + +func Test_decodePageToken(t *testing.T) { + encode := func(s string) string { + return base64.RawURLEncoding.EncodeToString([]byte(s)) + } + + cc := map[string]struct { + Token string + Result int + Err error + }{ + "Empty token": {Token: "", Result: 0}, + "Invalid base64": {Token: "%%%", Err: ErrInvalidPageToken}, + "Invalid JSON": {Token: encode("nope"), Err: ErrInvalidPageToken}, + "Negative offset": {Token: encode(`{"o":-1,"f":"fp"}`), Err: ErrInvalidPageToken}, + "Another fingerprint": {Token: encode(`{"o":1,"f":"other"}`), Err: ErrInvalidPageToken}, + "Valid token": {Token: encodePageToken(30, "fp"), Result: 30}, + } + + for cn, c := range cc { + t.Run(cn, func(t *testing.T) { + t.Parallel() + + res, err := decodePageToken(c.Token, "fp") + testutil.AssertEqualError(t, c.Err, err) + assert.Equal(t, c.Result, res) + }) + } +} + +func Test_groupHits(t *testing.T) { + documentID, branchID, forkID := xid.New(), xid.New(), xid.New() + + // hit builds a match carrying the fields the grouping reads. + hit := func(id, documentID, branchID, typ string) *bleveSearch.DocumentMatch { + return &bleveSearch.DocumentMatch{ + ID: id, + Fields: map[string]any{ + _fieldDocumentID: documentID, + _fieldBranchID: branchID, + _fieldType: typ, + }, + } + } + + cc := map[string]struct { + Hits bleveSearch.DocumentMatchCollection + Result []*groupEntries + Err error + }{ + "Invalid document id": { + Hits: bleveSearch.DocumentMatchCollection{hit("a", "nope", branchID.String(), "paragraph")}, + Err: assert.AnError, + }, + "Invalid branch id": { + Hits: bleveSearch.DocumentMatchCollection{hit("a", documentID.String(), "nope", "paragraph")}, + Err: assert.AnError, + }, + "No hits": {}, + "Hits grouped by branch in first appearance order": { + Hits: bleveSearch.DocumentMatchCollection{ + hit("a", documentID.String(), branchID.String(), "heading"), + hit("b", documentID.String(), forkID.String(), "paragraph"), + hit("c", documentID.String(), branchID.String(), "document"), + hit("d", documentID.String(), branchID.String(), "paragraph"), + }, + Result: []*groupEntries{ + {documentID: documentID, branchID: branchID, titleID: "c", hitIDs: []string{"a", "d"}}, + {documentID: documentID, branchID: forkID, hitIDs: []string{"b"}}, + }, + }, + } + + for cn, c := range cc { + t.Run(cn, func(t *testing.T) { + t.Parallel() + + res, err := groupHits(c.Hits) + testutil.AssertEqualError(t, c.Err, err) + assert.Equal(t, c.Result, res) + }) + } +}