github-mirror/oxynote
1
0
Fork 0
mirror of https://github.com/oxynote/oxynote.git synced 2026-10-17 02:49:18 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

replace meilisearch with belve

Browse Source
This commit is contained in:
david 2026-09-14 17:34:12 +03:00
parent 7142ca90b7
commit dca231af76
95 changed files with 4645 additions and 18403 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
AGENTS.md
View File

@ -139,9 +139,8 @@ Component commands are in the nested files.
auth-realtime `OXYNOTE_AUTH_REALTIME_*`, the frontend `NUXT_PUBLIC_*`.
- **`_DSN` vs `_URL`**: a connection string carrying credentials ends in
`_DSN`; a plain address ends in `_URL` and any credential is its own
variable (`MEILISEARCH_URL` + `_MASTER_KEY`). The suffix says whether the
value is a secret, so the same dependency is never `_DSN` on one side of
the trust boundary and `_URL` on the other.
variable (`CHANGEDETECTION_API_URL` + `_API_KEY`). The suffix says whether
the value is a secret and never changes across the trust boundary.
## Code style (TS/JS — web and auth-realtime)

7
CONTRIBUTING.md
View File

@ -49,9 +49,10 @@ picked up by simply running them again.
The dev stack reads `docker/env/*.local.env`, which `make setup` creates
from the committed `*.example.env` templates. The files are gitignored and
the defaults work out of the box. Integrations (GitHub app, Slack app, AI
assistant, email, changedetection.io, Meilisearch) are optional and stay
disabled until their variables are set; the frontend hides features the
deployment does not offer.
assistant, email, changedetection.io) are optional and stay disabled until
their variables are set; the frontend hides features the deployment does
not offer. Search is built in: core keeps its index on the container's
volume and rebuilds it from Postgres when it is missing.
```sh
make check-env # report variables that drifted between the templates and your local files

27
README.md
View File

@ -123,7 +123,7 @@ docker run -d --name oxynote \
Open http://localhost:8080 and sign up.
<details>
<summary>No PostgreSQL yet? This compose file starts one next to Oxynote, plus Meilisearch for search and Mailpit for email</summary>
<summary>No PostgreSQL yet? This compose file starts one next to Oxynote, plus Mailpit for email</summary>
```yaml
name: oxynote
@ -135,8 +135,6 @@ services:
- "8080:8080"
environment:
- OXYNOTE_DB_DSN=postgresql://oxynote:change-me-db@postgres/oxynote?sslmode=disable
- OXYNOTE_MEILISEARCH_URL=http://meilisearch:7700
- OXYNOTE_MEILISEARCH_MASTER_KEY=change-me-meili-key
# point these at a real relay to have mail delivered; here they go
# to the mailpit service below.
- OXYNOTE_SMTP_DSN=smtp://mailpit:1025?tls=none
@ -146,8 +144,6 @@ services:
depends_on:
postgres:
condition: service_healthy
meilisearch:
condition: service_healthy
mailpit:
condition: service_started
# the image shuts its services down in order, flushing open documents
@ -170,22 +166,6 @@ services:
retries: 5
restart: unless-stopped
# full-text search.
meilisearch:
image: getmeili/meilisearch:v1.53.1
environment:
- MEILI_ENV=production
- MEILI_MASTER_KEY=change-me-meili-key
- MEILI_NO_ANALYTICS=true
volumes:
- meilisearch_data:/meili_data
healthcheck:
test: ["CMD", "curl", "--fail", "http://localhost:7700/health"]
interval: 10s
timeout: 5s
retries: 5
restart: unless-stopped
# catches every mail Oxynote sends and shows it at
# http://localhost:8025.
mailpit:
@ -197,13 +177,12 @@ services:
volumes:
oxynote_data:
postgres_data:
meilisearch_data:
```
</details>
Everything else, from search and email to object storage, the GitHub
and Slack apps, social login and Rubber Duck, is one variable away and
Everything else, from email to object storage, the GitHub and Slack
apps, social login and Rubber Duck, is one variable away and
listed in [docker/prod/README.md](docker/prod/README.md). To run from
source, see [CONTRIBUTING.md](CONTRIBUTING.md).

32
docker/docker-compose.dev.yaml
View File

@ -70,15 +70,14 @@ services:
depends_on:
postgres:
condition: service_healthy
meilisearch:
condition: service_healthy
volumes:
# optional GitHub App private key: drop oxynote.private-key.pem into
# docker/github/ and set OXYNOTE_CORE_GITHUB_APP_ID to enable the
# GitHub App integration.
- ./github:/etc/oxynote/core/github:ro
# uploaded files, which core keeps under OXYNOTE_CORE_OBJECT_STORAGE_LOCAL_PATH
# when no object store is configured. The mount is the home directory
# when no object store is configured, and the search index under
# OXYNOTE_CORE_SEARCH_INDEX_PATH. The mount is the home directory
# rather than the storage path itself: core runs as a non-root uid
# that owns only /home/nonroot in its image, and an empty volume
# takes the ownership of the directory it is mounted over.
@ -153,32 +152,6 @@ services:
mode: non-blocking
max-size: 10m
max-file: "3"
meilisearch:
container_name: meilisearch
image: getmeili/meilisearch:v1.53.2
volumes:
- meilisearch_data:/meili_data
environment:
- MEILI_MASTER_KEY=devmasterkey
- MEILI_ENV=development
- MEILI_LOG_LEVEL=WARN
- MEILI_NO_ANALYTICS=true
# the data volume outlives image bumps, and meilisearch refuses to
# open a database written by an older engine unless told to upgrade
# it in place
- MEILI_UPGRADE_DB=true
healthcheck:
test: ["CMD", "curl", "--fail", "http://localhost:7700/health"]
interval: 5s
timeout: 5s
retries: 20
restart: unless-stopped
logging:
driver: local
options:
mode: non-blocking
max-size: 10m
max-file: "3"
mailpit:
container_name: mailpit
image: axllent/mailpit:v1.31.1
@ -260,7 +233,6 @@ volumes:
core_storage:
mariadb_data:
postgres_data:
meilisearch_data:
valkey_data:
caddy_data:
caddy_config:

10
docker/env/core.example.env vendored
View File

@ -5,10 +5,12 @@
# infrastructure wiring (compose service hostnames)
OXYNOTE_CORE_DB_DSN=postgresql://devuser:devpass@postgres/devdb?sslmode=disable
# an empty DSN disables search: documents are not indexed, the search
# endpoint refuses, and the assistant loses its search tool.
OXYNOTE_CORE_MEILISEARCH_URL=http://meilisearch:7700
OXYNOTE_CORE_MEILISEARCH_MASTER_KEY=devmasterkey
# where the search index is kept. It is a cache of the database rebuilt
# from it when missing, so it needs no backup, but it pins a deployment to
# one core instance. Core has no default for it and refuses to boot when
# it is unset. It sits next to the object storage path on the volume the
# dev stack mounts over /home/nonroot.
OXYNOTE_CORE_SEARCH_INDEX_PATH=/home/nonroot/search
# an empty URL disables valkey: the assistant keeps conversation history,
# checkpoints and pending confirmations in the core process instead.
# redis[s]://[username:password@]host:port[/db]

5
docker/prod/AGENTS.md
View File

@ -8,9 +8,10 @@ principles and TS/JS style live in the root [AGENTS.md](../../AGENTS.md).
One public image (`ghcr.io/oxynote/oxynote`) running Caddy, web (Nuxt SSR),
core and auth-realtime under `launcher/` (`@oxynote/launcher`, pnpm), a
TypeScript supervisor run by node under tini. Postgres is the only required
external service; valkey, an S3-compatible store, meilisearch, SMTP and
external service; valkey, an S3-compatible store, SMTP and
changedetection.io are optional (without a store, objects live on the data
volume). `docker-compose.example.yaml` is the canonical deployment and
volume, as does the search index, which core rebuilds from Postgres when
missing). `docker-compose.example.yaml` is the canonical deployment and
`README.md` the operator reference. `docker-compose.local.yaml` is the
override `make prod-run` layers on it to build the image locally and add a
mailpit, since signup needs a verified address.

7
docker/prod/README.md
View File

@ -3,8 +3,10 @@
One container running the whole product: Caddy (front door), the web app
(Nuxt SSR), the core API server, and the auth/realtime service, supervised by
a small launcher. PostgreSQL runs outside the image and is the only thing
it requires; Valkey, an S3-compatible object store, Meilisearch, an SMTP
relay and changedetection.io are optional and also external.
it requires; Valkey, an S3-compatible object store, an SMTP relay and
changedetection.io are optional and also external. Full-text search is
built in: core keeps its index on the data volume and rebuilds it from
PostgreSQL when it is missing, so the image runs as a single instance.
## Quick start
@ -53,7 +55,6 @@ cleanly. An incomplete group fails the boot with the missing name.
| --- | --- |
| Valkey/Redis | `OXYNOTE_VALKEY_DSN` (`redis[s]://[user:pass@]host:port[/db]`, credentials supported). Without it the assistant keeps its conversations in the core process and sessions are not cached outside Postgres. |
| Object storage | `OXYNOTE_OBJECT_STORAGE_DSN` — an S3-compatible store as one URL: `http(s)://ACCESS_KEY:SECRET_KEY@host:port/bucket[?region=...]` (the bucket defaults to `oxynote` and is created if missing). Without it uploaded images are kept on the data volume, which is the right choice for a single-node deployment. |
| Search | `OXYNOTE_MEILISEARCH_URL`, `OXYNOTE_MEILISEARCH_MASTER_KEY` |
| Email | `OXYNOTE_SMTP_DSN` (`smtp[s]://[user:pass@]host:port[?tls=none\|starttls\|tls]`), `OXYNOTE_EMAIL_FROM_ADDRESS`. Without email, verification mails are logged instead of sent. |
| GitHub App | `OXYNOTE_GITHUB_APP_ID`, `OXYNOTE_GITHUB_APP_SLUG`, `OXYNOTE_GITHUB_APP_SIGNATURE_SECRET`; mount the app's private key at `/oxynote/github/private-key.pem` |
| Slack app | `OXYNOTE_SLACK_APP_CLIENT_ID`, `OXYNOTE_SLACK_APP_CLIENT_SECRET`, `OXYNOTE_SLACK_APP_SIGNATURE_SECRET` |

29
docker/prod/docker-compose.example.yaml
View File

@ -22,24 +22,20 @@ services:
# the default, and it matches the 8080:8080 port mapping above.
- OXYNOTE_PUBLIC_URL=http://localhost:8080
- OXYNOTE_DB_DSN=postgresql://oxynote:change-me-db@postgres/oxynote?sslmode=disable
# optional: full-text search.
- OXYNOTE_MEILISEARCH_URL=http://meilisearch:7700
- OXYNOTE_MEILISEARCH_MASTER_KEY=change-me-meili-key
# optional: outgoing email (signup verification, invites). Point it
# at a real relay, or uncomment the mailpit service below for a
# trial run and use these two lines:
# - OXYNOTE_SMTP_DSN=smtp://mailpit:1025
# - OXYNOTE_EMAIL_FROM_ADDRESS=Oxynote <team@example.com>
volumes:
# generated secrets and every uploaded image live here. Losing this
# volume signs everyone out, takes the images with it and
# permanently orphans stored data-source credentials.
# generated secrets, every uploaded image and the search index live
# here. Losing this volume signs everyone out, takes the images with
# it and permanently orphans stored data-source credentials; the
# index alone is rebuilt from the database at the next start.
- oxynote_data:/oxynote/data
depends_on:
postgres:
condition: service_healthy
meilisearch:
condition: service_healthy
# the image shuts its services down in order (flushing open documents
# last); docker's default 10s grace would SIGKILL that mid-flush.
stop_grace_period: 60s
@ -61,22 +57,6 @@ services:
retries: 5
restart: unless-stopped
meilisearch:
image: getmeili/meilisearch:v1.53.2
environment:
- MEILI_ENV=production
- MEILI_MASTER_KEY=change-me-meili-key
- MEILI_LOG_LEVEL=WARN
- MEILI_NO_ANALYTICS=true
volumes:
- meilisearch_data:/meili_data
healthcheck:
test: ["CMD", "curl", "--fail", "http://localhost:7700/health"]
interval: 10s
timeout: 5s
retries: 5
restart: unless-stopped
# a trial-run mail sink: uncomment it together with the two OXYNOTE_SMTP
# variables above and read the delivered mail at http://localhost:8025.
# mailpit:
@ -87,4 +67,3 @@ services:
volumes:
oxynote_data:
postgres_data:
meilisearch_data:

21
docker/prod/launcher/src/env.test.ts
View File

@ -320,19 +320,6 @@ describe("loadConfig", () => {
})
describe("feature groups", () => {
it("requires the meilisearch master key with the URL", ({
expect,
}) => {
expect(() =>
loadConfig(
completeEnv({
OXYNOTE_MEILISEARCH_URL:
"http://meilisearch:7700",
}),
),
).toThrow("OXYNOTE_MEILISEARCH_MASTER_KEY")
})
it("rejects a companion without its keying variable", ({
expect,
}) => {
@ -393,10 +380,6 @@ describe("loadConfig", () => {
}) => {
const config = loadConfig(
completeEnv({
OXYNOTE_MEILISEARCH_URL:
"http://meilisearch:7700",
OXYNOTE_MEILISEARCH_MASTER_KEY:
"master",
OXYNOTE_CHANGE_DETECTION_URL:
"http://changeDetection:5000",
OXYNOTE_GITHUB_APP_ID: "123",
@ -413,10 +396,6 @@ describe("loadConfig", () => {
}),
)
expect(config.meilisearch).toEqual({
url: "http://meilisearch:7700",
masterKey: "master",
})
expect(config.changeDetection).toEqual({
url: "http://changeDetection:5000",
apiKey: "",

16
docker/prod/launcher/src/env.ts
View File

@ -64,7 +64,6 @@ export interface Config {
objectStorage: ObjectStorageConfig | undefined
smtp: SmtpConfig | undefined
emailFromAddress: string
meilisearch: { url: string; masterKey: string } | undefined
changeDetection: { url: string; apiKey: string } | undefined
githubApp: GitHubAppConfig | undefined
slackApp: SlackAppConfig | undefined
@ -339,8 +338,6 @@ const baseSchema = z.object({
OXYNOTE_OBJECT_STORAGE_DSN: objectStorageDsn.optional(),
OXYNOTE_SMTP_DSN: smtpDsn.optional(),
OXYNOTE_EMAIL_FROM_ADDRESS: z.string().optional(),
OXYNOTE_MEILISEARCH_URL: httpUrl().optional(),
OXYNOTE_MEILISEARCH_MASTER_KEY: z.string().optional(),
OXYNOTE_CHANGE_DETECTION_URL: httpUrl().optional(),
OXYNOTE_CHANGE_DETECTION_API_KEY: z.string().optional(),
OXYNOTE_GITHUB_APP_ID: z
@ -404,11 +401,6 @@ const featureGroups: {
requires: ["OXYNOTE_EMAIL_FROM_ADDRESS"],
companions: ["OXYNOTE_EMAIL_FROM_ADDRESS"],
},
{
key: "OXYNOTE_MEILISEARCH_URL",
requires: ["OXYNOTE_MEILISEARCH_MASTER_KEY"],
companions: ["OXYNOTE_MEILISEARCH_MASTER_KEY"],
},
{
key: "OXYNOTE_CHANGE_DETECTION_URL",
requires: [],
@ -576,14 +568,6 @@ export function loadConfig(source: Record<string, string | undefined>): Config {
objectStorage: values.OXYNOTE_OBJECT_STORAGE_DSN,
smtp: values.OXYNOTE_SMTP_DSN,
emailFromAddress: values.OXYNOTE_EMAIL_FROM_ADDRESS ?? "",
meilisearch: values.OXYNOTE_MEILISEARCH_URL
? {
url: values.OXYNOTE_MEILISEARCH_URL,
masterKey:
values.OXYNOTE_MEILISEARCH_MASTER_KEY ??
"",
}
: undefined,
changeDetection: values.OXYNOTE_CHANGE_DETECTION_URL
? {
url: values.OXYNOTE_CHANGE_DETECTION_URL,

3
docker/prod/launcher/src/index.ts
View File

@ -131,8 +131,7 @@ function logEnabledFeatures(config: Config): void {
const state = (enabled: boolean) => (enabled ? "on" : "off")
log.info(
`search ${state(config.meilisearch !== undefined)}, ` +
`email ${state(config.smtp !== undefined)}, ` +
`email ${state(config.smtp !== undefined)}, ` +
`github app ${state(config.githubApp !== undefined)}, ` +
`slack app ${state(config.slackApp !== undefined)}, ` +
`ai assistant ${state((config.aiAssistant.PROVIDER ?? "") !== "")}, ` +

18
docker/prod/launcher/src/mapping.test.ts
View File

@ -194,6 +194,16 @@ describe("buildChildEnvs", () => {
expect(envs.core.OXYNOTE_CORE_OBJECT_STORAGE_BUCKET).toBe("")
})
// the index is a cache of the database, kept on the volume so a restart
// does not rebuild it; operators never point it elsewhere
it("keeps the search index on the data volume", ({ expect }) => {
const envs = build()
expect(envs.core.OXYNOTE_CORE_SEARCH_INDEX_PATH).toBe(
"/oxynote/data/search",
)
})
it("keeps the storage path alongside a configured object store", ({
expect,
}) => {
@ -248,7 +258,6 @@ describe("buildChildEnvs", () => {
it("leaves a disabled feature's variables empty", ({ expect }) => {
const envs = build()
expect(envs.core.OXYNOTE_CORE_MEILISEARCH_URL).toBe("")
expect(envs.core.OXYNOTE_CORE_EMAIL_SMTP_HOST).toBe("")
expect(envs.core.OXYNOTE_CORE_GITHUB_APP_ID).toBe("")
expect(envs.core.OXYNOTE_CORE_SLACK_CLIENT_ID).toBe("")
@ -272,10 +281,6 @@ describe("buildChildEnvs", () => {
tls: "starttls",
},
emailFromAddress: "Oxynote <team@example.com>",
meilisearch: {
url: "http://meilisearch:7700",
masterKey: "master",
},
githubApp: {
appId: "123",
appSlug: "oxynote",
@ -309,9 +314,6 @@ describe("buildChildEnvs", () => {
expect(envs.core.OXYNOTE_CORE_EMAIL_FROM_ADDRESS).toBe(
"Oxynote <team@example.com>",
)
expect(envs.core.OXYNOTE_CORE_MEILISEARCH_MASTER_KEY).toBe(
"master",
)
expect(envs.core.OXYNOTE_CORE_GITHUB_APP_ID).toBe("123")
expect(envs.core.OXYNOTE_CORE_GITHUB_PRIVATE_KEY_PATH).toBe(
"/oxynote/github/private-key.pem",

9
docker/prod/launcher/src/mapping.ts
View File

@ -24,6 +24,11 @@ export const dataDir = "/oxynote/data"
// secrets do.
const objectStorageLocalPath = `${dataDir}/object-storage`
// where core keeps the search index. It is a cache of the database that
// core rebuilds when it is missing, and it sits on the data volume so a
// restart does not pay for that rebuild.
const searchIndexPath = `${dataDir}/search`
// the fixed path a GitHub App's private key is mounted at when the
// integration is enabled.
const githubPrivateKeyPath = "/oxynote/github/private-key.pem"
@ -95,9 +100,7 @@ export function buildChildEnvs(
OXYNOTE_CORE_SERVER_ADDRESS: `127.0.0.1:${corePort}`,
OXYNOTE_CORE_DB_DSN: config.databaseDsn,
OXYNOTE_CORE_VALKEY_DSN: config.valkeyDsn ?? "",
OXYNOTE_CORE_MEILISEARCH_URL: config.meilisearch?.url ?? "",
OXYNOTE_CORE_MEILISEARCH_MASTER_KEY:
config.meilisearch?.masterKey ?? "",
OXYNOTE_CORE_SEARCH_INDEX_PATH: searchIndexPath,
// an empty URL is what makes core store objects on disk at
// OBJECT_STORAGE_LOCAL_PATH instead of in a remote store.
OXYNOTE_CORE_OBJECT_STORAGE_URL:

1
docker/prod/launcher/src/test-helpers.ts
View File

@ -35,7 +35,6 @@ export function testConfig(overrides: Partial<Config> = {}): Config {
},
smtp: undefined,
emailFromAddress: "",
meilisearch: undefined,
changeDetection: undefined,
githubApp: undefined,
slackApp: undefined,

18
e2e/docker-compose.dev.yaml
View File

@ -105,8 +105,7 @@ services:
image: ghcr.io/oxynote/core:e2e-dev
environment:
- OXYNOTE_CORE_DB_DSN=postgresql://devuser:devpass@postgres/devdb?sslmode=disable
- OXYNOTE_CORE_MEILISEARCH_URL=http://meilisearch:7700
- OXYNOTE_CORE_MEILISEARCH_MASTER_KEY=devmasterkey
- OXYNOTE_CORE_SEARCH_INDEX_PATH=/home/nonroot/search
- OXYNOTE_CORE_OBJECT_STORAGE_LOCAL_PATH=/home/nonroot/storage
- OXYNOTE_CORE_SERVER_AUTH_BETTER_AUTH_URL=http://auth-realtime:8081/api/auth/get-session
# the MCP surface: core validates bearer tokens against auth-realtime
@ -127,8 +126,6 @@ services:
depends_on:
postgres:
condition: service_healthy
meilisearch:
condition: service_healthy
postgres:
image: postgres:18.6-alpine
@ -143,19 +140,6 @@ services:
timeout: 5s
retries: 20
meilisearch:
image: getmeili/meilisearch:v1.53.2
environment:
- MEILI_MASTER_KEY=devmasterkey
- MEILI_ENV=development
- MEILI_LOG_LEVEL=WARN
- MEILI_NO_ANALYTICS=true
healthcheck:
test: ["CMD", "curl", "--fail", "http://localhost:7700/health"]
interval: 5s
timeout: 5s
retries: 20
# the tests read verification links out of mailpit's REST API, so its
# http port is published rather than proxied through caddy.
mailpit:

16
e2e/docker-compose.prod.yaml
View File

@ -35,8 +35,6 @@ services:
environment:
- OXYNOTE_PUBLIC_URL=http://localhost:19080
- OXYNOTE_DB_DSN=postgresql://devuser:devpass@postgres/devdb?sslmode=disable
- OXYNOTE_MEILISEARCH_URL=http://meilisearch:7700
- OXYNOTE_MEILISEARCH_MASTER_KEY=devmasterkey
- OXYNOTE_SMTP_DSN=smtp://mailpit:1025?tls=none
- OXYNOTE_EMAIL_FROM_ADDRESS=Oxynote <team@oxynote.io>
- OXYNOTE_MAX_ORGANIZATION_MEMBERS=5
@ -51,8 +49,6 @@ services:
depends_on:
postgres:
condition: service_healthy
meilisearch:
condition: service_healthy
# the image ships its own layered probe (every internal service plus the
# front door), which is what the probe container and the tests wait on.
# Its start period is three minutes; compose inherits that.
@ -89,18 +85,6 @@ services:
timeout: 5s
retries: 20
meilisearch:
image: getmeili/meilisearch:v1.53.2
environment:
- MEILI_MASTER_KEY=devmasterkey
- MEILI_ENV=development
- MEILI_LOG_LEVEL=WARN
healthcheck:
test: ["CMD", "curl", "--fail", "http://localhost:7700/health"]
interval: 5s
timeout: 5s
retries: 20
# the tests read verification links out of mailpit's REST API, so its http
# port is published rather than proxied through the image's front door —
# which routes nothing but the product.

4
e2e/tests/search.test.ts
View File

@ -23,8 +23,8 @@ import { signUpWithWorkspace } from "../helpers/workspace"
test.describe("search", () => {
// on top of the usual setup, a find waits out the storage debounce
// and the ten-second search-indexing pass before the first hit can
// appear. slow() triples the budget for that chain.
// before the first hit can appear; the index itself follows the
// persist directly. slow() triples the budget for that chain.
test.beforeEach(() => {
test.slow()
})

29
server/AGENTS.md
View File

@ -11,8 +11,8 @@ auth-realtime: [auth-realtime/AGENTS.md](auth-realtime/AGENTS.md).
## Stack
- `server/core/`: one binary, `cmd/core`, on `:8080`. Owns Postgres,
Meilisearch, Valkey, object storage, the GitHub/Slack apps, the assistant
- `server/core/`: one binary, `cmd/core`, on `:8080`. Owns Postgres, the
search index, Valkey, object storage, the GitHub/Slack apps, the assistant
and outbound data-source connections.
- `server/auth-realtime/`: `:8081`, Better Auth (organization plugin) and
Hocuspocus in one Hono process; forwards non-auth `/api/...` to core
@ -39,9 +39,9 @@ GitHub and Slack callback URLs point at `:8080/core/api/apps/...`.
- `/api/...`: session-authed via auth-realtime's `/api/auth/get-session`
(`SERVER_AUTH_BETTER_AUTH_URL`). `GET /api/capabilities` reports one
boolean per optional service (`github`, `slack`, `changeDetection`,
`search`), snapshotted at boot from each client's `Configured()`, plus
`aiAssistant` (`status` + `model`).
boolean per optional service (`github`, `slack`, `changeDetection`),
snapshotted at boot from each client's `Configured()`, plus `aiAssistant`
(`status` + `model`).
- `/api/x/...`: no auth; auth-realtime fetches/stores branch content here
(`/x/documents/{id}/branches`, `/x/documents/{id}/branch/{branchId}`),
triggers emails and initializes or tears down orgs.
@ -119,12 +119,13 @@ ones included; the dev compose reads only the gitignored `*.local.env`
copies. Core reads `OXYNOTE_CORE_*` through `buildinfo.Getenv("FOO")`.
Every integration is optional and keyed on one variable (`GITHUB_APP_ID`,
`SLACK_CLIENT_ID`, `ASSISTANT_PROVIDER`, `MEILISEARCH_DSN`,
`CHANGEDETECTION_API_URL`, `VALKEY_ADDRESS`, `S3_URL`, `EMAIL_SMTP_HOST`).
Key set with the rest of its group missing is a boot error; key empty
disables the feature (routes answer `<domain>.not_configured`, background
work skipped). Two degrade instead and confine the deployment to one
instance: empty `S3_URL` stores objects under `S3_LOCAL_PATH` (no default;
unset is a boot error), empty `VALKEY_ADDRESS` keeps assistant conversations
in-process (`pkg/memkit`). A model too weak for the assistant disables it
with a warning.
`SLACK_CLIENT_ID`, `ASSISTANT_PROVIDER`, `CHANGEDETECTION_API_URL`,
`VALKEY_ADDRESS`, `S3_URL`, `EMAIL_SMTP_HOST`). Key set with the rest of its
group missing is a boot error; key empty disables the feature (routes answer
`<domain>.not_configured`, background work skipped). Two degrade instead:
empty `S3_URL` stores objects under `S3_LOCAL_PATH` (no default; unset is a
boot error), empty `VALKEY_ADDRESS` keeps assistant conversations in-process
(`pkg/memkit`). A model too weak for the assistant disables it with a
warning. Search: `SEARCH_INDEX_PATH` (no default) is a bleve index, a
Postgres cache rebuilt at boot when missing or outdated; it pins a
deployment to one core instance.

4
server/core/AGENTS.md
View File

@ -188,8 +188,8 @@ pagination through `sqlutil.Select`; empty results are a non-nil empty
slice; always a deterministic tiebreaker after the user sort. IDs are
generated in the domain layer, no `RETURNING`. Soft deletes set
`deleted_at`; hard deletes happen in `Cleanup*` sweeps. A sqlhooks error
hook maps `sql.ErrNoRows` → `errutil.ErrNotFound` and constraint violations
→ user errors, so callers only `errors.Is`. Migrations are
hook maps constraint violations → user errors; an empty result is a bare
`sql.ErrNoRows`, so callers use `errutil.IsNotFound`. Migrations are
`migrations/NNN_snake_case.sql`, embedded with `//go:embed`.
## HTTP & WebSocket

39
server/core/cmd/core/main.go
View File

@ -19,7 +19,6 @@ import (
"github.com/cloudwego/eino/components/model"
"github.com/gomodule/redigo/redis"
"github.com/jellydator/xync"
"github.com/meilisearch/meilisearch-go"
"github.com/oxynote/oxynote/server/core/internal/apps/github"
"github.com/oxynote/oxynote/server/core/internal/apps/slack"
"github.com/oxynote/oxynote/server/core/internal/apps/webchange"
@ -205,25 +204,18 @@ func main() { //nolint:maintidx // main performs linear wiring of all components
termCtx, termCancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer termCancel()
// an empty MEILISEARCH_URL means search is disabled: nothing is
// indexed and the search surfaces refuse. A set URL that cannot be
// reached stays a boot error inside NewClient.
var meiliMan meilisearch.ServiceManager
if url := buildinfo.Getenv("MEILISEARCH_URL"); url != "" {
meiliMan = meilisearch.New(
url,
meilisearch.WithAPIKey(buildinfo.Getenv("MEILISEARCH_MASTER_KEY")),
)
}
searchClient, err := search.NewClient(termCtx, meiliMan)
// the index is a cache of the database kept on local disk, which is
// what confines a deployment to one core instance. Opening it builds
// it from the database when it is missing or outdated.
searchIndex, err := search.Open(termCtx, log, buildinfo.Getenv("SEARCH_INDEX_PATH"), dbc)
if err != nil {
fail(log, closers, "cannot create search gateway client", err)
fail(log, closers, "cannot open the search index", err)
return
}
searchJobs := search.NewJobs(searchClient.Configured())
closers = append([]io.Closer{searchIndex}, closers...)
searchJobMan := searchMan.NewManager(log, dbc, searchIndex)
objectStorageURL := buildinfo.Getenv("OBJECT_STORAGE_URL")
@ -293,8 +285,8 @@ func main() { //nolint:maintidx // main performs linear wiring of all components
summaryModel,
metrics,
editClient,
searchClient,
searchJobs,
searchIndex,
searchJobMan,
datasourceMan,
githubMan,
webchangeClient,
@ -307,7 +299,6 @@ func main() { //nolint:maintidx // main performs linear wiring of all components
warnDisabled(log, githubMan.Configured(), "github app integration is disabled")
warnDisabled(log, slackMan.Configured(), "slack app integration is disabled")
warnDisabled(log, assistantMan.Configured(), "assistant is disabled")
warnDisabled(log, searchClient.Configured(), "search is disabled")
warnDisabled(log, webchangeClient.Configured(), "changedetection integration is disabled")
serverHost, serverPort, err := parseServerAddress(buildinfo.Getenv("SERVER_ADDRESS"))
@ -344,8 +335,8 @@ func main() { //nolint:maintidx // main performs linear wiring of all components
githubMan,
slackMan,
webchangeClient,
searchClient,
searchJobs,
searchIndex,
searchJobMan,
notifMan,
emailSender,
http.DefaultClient,
@ -389,11 +380,7 @@ func main() { //nolint:maintidx // main performs linear wiring of all components
backgroundSupv.Go(assistantMan.Start)
backgroundSupv.Go(simulationChecker.Start)
// without search there are no queued jobs to drain, so the manager
// is not started at all.
if searchClient.Configured() {
backgroundSupv.Go(searchMan.NewManager(log, dbc, searchClient).Start)
}
backgroundSupv.Go(searchJobMan.Start)
<-termCtx.Done()

26
server/core/go.mod
View File

@ -9,6 +9,8 @@ require (
github.com/aws/aws-sdk-go-v2/credentials v1.20.3
github.com/aws/aws-sdk-go-v2/service/s3 v1.112.0
github.com/blang/semver/v4 v4.0.0
github.com/blevesearch/bleve/v2 v2.6.1
github.com/blevesearch/bleve_index_api v1.4.1
github.com/bradleyfalzon/ghinstallation/v2 v2.19.0
github.com/cenkalti/backoff/v4 v4.3.0
github.com/cloudwego/eino v0.9.19
@ -38,7 +40,6 @@ require (
github.com/jellydator/xync v0.0.0-20240601154136-0f038e166df4
github.com/jmoiron/sqlx v1.4.0
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0
github.com/meilisearch/meilisearch-go v0.36.3
github.com/modelcontextprotocol/go-sdk v1.7.0
github.com/orlangure/gnomock v0.32.1-0.20260413153725-5ce19d18d84a
github.com/oxynote/wetsocks v0.1.0
@ -67,7 +68,7 @@ require (
cloud.google.com/go/compute/metadata v0.9.0 // indirect
filippo.io/edwards25519 v1.2.0 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/andybalholm/brotli v1.2.1 // indirect
github.com/RoaringBitmap/roaring/v2 v2.14.5 // indirect
github.com/anthropics/anthropic-sdk-go v1.56.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect
github.com/aws/aws-sdk-go-v2/config v1.32.31 // indirect
@ -86,6 +87,24 @@ require (
github.com/aws/smithy-go v1.28.1 // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.24.2 // indirect
github.com/blevesearch/geo v0.2.6 // indirect
github.com/blevesearch/go-faiss v1.1.5 // indirect
github.com/blevesearch/go-porterstemmer v1.0.3 // indirect
github.com/blevesearch/gtreap v0.1.1 // indirect
github.com/blevesearch/mmap-go v1.2.0 // indirect
github.com/blevesearch/scorch_segment_api/v2 v2.4.10 // indirect
github.com/blevesearch/segment v0.9.1 // indirect
github.com/blevesearch/snowballstem v0.9.0 // indirect
github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect
github.com/blevesearch/vellum v1.2.0 // indirect
github.com/blevesearch/zapx/v11 v11.4.3 // indirect
github.com/blevesearch/zapx/v12 v12.4.3 // indirect
github.com/blevesearch/zapx/v13 v13.4.3 // indirect
github.com/blevesearch/zapx/v14 v14.4.3 // indirect
github.com/blevesearch/zapx/v15 v15.4.3 // indirect
github.com/blevesearch/zapx/v16 v16.3.4 // indirect
github.com/blevesearch/zapx/v17 v17.2.3 // indirect
github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect
github.com/buger/jsonparser v1.2.0 // indirect
github.com/bytedance/gopkg v0.1.3 // indirect
@ -115,6 +134,7 @@ require (
github.com/go-logr/stdr v1.2.2 // indirect
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
github.com/golang/snappy v1.0.0 // indirect
github.com/google/go-querystring v1.2.0 // indirect
github.com/google/jsonschema-go v0.4.3 // indirect
github.com/google/s2a-go v0.1.9 // indirect
@ -143,6 +163,7 @@ require (
github.com/moby/moby/client v0.5.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
github.com/mschoch/smat v0.2.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
github.com/nikolalohinski/gonja v1.5.3 // indirect
@ -165,6 +186,7 @@ require (
github.com/x448/float16 v0.8.4 // indirect
github.com/yargevad/filepathx v1.0.0 // indirect
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
go.etcd.io/bbolt v1.4.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect

52
server/core/go.sum
View File

@ -23,11 +23,11 @@ github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8
github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/RoaringBitmap/roaring/v2 v2.14.5 h1:ckd0o545JqDPeVJDgeFoaM21eBixUnlWfYgjE5VnyWw=
github.com/RoaringBitmap/roaring/v2 v2.14.5/go.mod h1:eq4wdNXxtJIS/oikeCzdX1rBzek7ANzbth041hrU8Q4=
github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZU11AGBXMILnrdOU8Kn00o=
github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0=
github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs=
github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
github.com/anthropics/anthropic-sdk-go v1.56.0 h1:idVU14wOZ06D0GBNEvuhn927xXmBVEquo0469iDwLsc=
github.com/anthropics/anthropic-sdk-go v1.56.0/go.mod h1:3EfIfmFqxH6rbiLcIP4tPFyXL/IHakx2wDG4OU+TIEI=
github.com/aws/aws-sdk-go-v2 v1.46.0 h1:1kt7m/EKcEHt5mlyyxx9cSlMddRPIKbjb6DIQsu4HPk=
@ -73,8 +73,48 @@ github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3/go.mod h1:CIWtjk
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
github.com/bits-and-blooms/bitset v1.24.2 h1:M7/NzVbsytmtfHbumG+K2bremQPMJuqv1JD3vOaFxp0=
github.com/bits-and-blooms/bitset v1.24.2/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
github.com/blevesearch/bleve/v2 v2.6.1 h1:47vLskRTqxvQEtxVPYHjf5KpOgzD2msslXFjvUQCgWQ=
github.com/blevesearch/bleve/v2 v2.6.1/go.mod h1:Dvvx6ZoEBTOj6RSzfk0lEz0wce/qhe2yOUubXeuzd2c=
github.com/blevesearch/bleve_index_api v1.4.1 h1:CYIyecFlI+/RYjzUm+NmDjYbSvk870Bb7f+Vl4b12q8=
github.com/blevesearch/bleve_index_api v1.4.1/go.mod h1:xvd48t5XMeeioWQ5/jZvgLrV98flT2rdvEJ3l/ki4Ko=
github.com/blevesearch/geo v0.2.6 h1:7K1oyQKYlauC+mJuo2AfNPyjN/4mihEoJMfyClVH1Mo=
github.com/blevesearch/geo v0.2.6/go.mod h1:6qzVUiB4BK47QkSZcRqiXEP2W3EeXuzM5XFTF8AdZ8A=
github.com/blevesearch/go-faiss v1.1.5 h1:/IU5lkOahH9Ghfk9n3F6N0XD7PYVXZJWmNDc9TtXuco=
github.com/blevesearch/go-faiss v1.1.5/go.mod h1:w3W9AiWsFRGVaMG+/cmJi7iHEAuGyC6blsgO1EzCK/M=
github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M=
github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y=
github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk=
github.com/blevesearch/mmap-go v1.2.0 h1:l33nNKPFcBjJUMwem6sAYJPUzhUCABoK9FxZDGiFNBI=
github.com/blevesearch/mmap-go v1.2.0/go.mod h1:Vd6+20GBhEdwJnU1Xohgt88XCD/CTWcqbCNxkZpyBo0=
github.com/blevesearch/scorch_segment_api/v2 v2.4.10 h1:C3873+iWZ0YJM2ijaSHhJJzSvD4x1k+5UaQdGygZVhM=
github.com/blevesearch/scorch_segment_api/v2 v2.4.10/go.mod h1:WUUkAocbkDlNK/kgAE13NvS9oxe+u618mYZ8sOvcCc4=
github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU=
github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw=
github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s=
github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs=
github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMGZzVrdmaozG2MfoB+A=
github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ=
github.com/blevesearch/vellum v1.2.0 h1:xkDiOEsHc2t3Cp0NsNZZ36pvc130sCzcGKOPMzXe+e0=
github.com/blevesearch/vellum v1.2.0/go.mod h1:uEcfBJz7mAOf0Kvq6qoEKQQkLODBF46SINYNkZNae4k=
github.com/blevesearch/zapx/v11 v11.4.3 h1:PTZOO5loKpHC/x/GzmPZNa9cw7GZIQxd5qRjwij9tHY=
github.com/blevesearch/zapx/v11 v11.4.3/go.mod h1:4gdeyy9oGa/lLa6D34R9daXNUvfMPZqUYjPwiLmekwc=
github.com/blevesearch/zapx/v12 v12.4.3 h1:eElXvAaAX4m04t//CGBQAtHNPA+Q6A1hHZVrN3LSFYo=
github.com/blevesearch/zapx/v12 v12.4.3/go.mod h1:TdFmr7afSz1hFh/SIBCCZvcLfzYvievIH6aEISCte58=
github.com/blevesearch/zapx/v13 v13.4.3 h1:qsdhRhaSpVnqDFlRiH9vG5+KJ+dE7KAW9WyZz/KXAiE=
github.com/blevesearch/zapx/v13 v13.4.3/go.mod h1:knK8z2NdQHlb5ot/uj8wuvOq5PhDGjNYQQy0QDnopZk=
github.com/blevesearch/zapx/v14 v14.4.3 h1:GY4Hecx0C6UTmiNC2pKdeA2rOKiLR5/rwpU9WR51dgM=
github.com/blevesearch/zapx/v14 v14.4.3/go.mod h1:rz0XNb/OZSMjNorufDGSpFpjoFKhXmppH9Hi7a877D8=
github.com/blevesearch/zapx/v15 v15.4.3 h1:iJiMJOHrz216jyO6lS0m9RTCEkprUnzvqAI2lc/0/CU=
github.com/blevesearch/zapx/v15 v15.4.3/go.mod h1:1pssev/59FsuWcgSnTa0OeEpOzmhtmr/0/11H0Z8+Nw=
github.com/blevesearch/zapx/v16 v16.3.4 h1:hDAqA8qusZTNbPEL7//w5P65UZ2de6yhSeUaTbp0Po0=
github.com/blevesearch/zapx/v16 v16.3.4/go.mod h1:zqkPPqs9GS9FzVWzCO3Wf1X044yWAV17+4zb+FTiEHg=
github.com/blevesearch/zapx/v17 v17.2.3 h1:UYYJPAt5b2tVxldx5h0jmv23RMsg8/UZKFVya7v92po=
github.com/blevesearch/zapx/v17 v17.2.3/go.mod h1:r7mb4QWbDQSkbAnOjCb9iCfkcrzajB4yBdJpuBIo/fE=
github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6TYWexEs=
github.com/bmatcuk/doublestar/v4 v4.10.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
@ -336,8 +376,6 @@ github.com/maxatome/go-testdeep v1.14.0 h1:rRlLv1+kI8eOI3OaBXZwb3O7xY3exRzdW5QyX
github.com/maxatome/go-testdeep v1.14.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM=
github.com/meguminnnnnnnnn/go-openai v0.1.2 h1:iXombGGjqjBrmE9WaSidUhhi3YQhf42QTHvHLMkgvCA=
github.com/meguminnnnnnnnn/go-openai v0.1.2/go.mod h1:qs96ysDmxhE4BZoU45I43zcyfnaYxU3X+aRzLko/htY=
github.com/meilisearch/meilisearch-go v0.36.3 h1:Yx1aTY5jDgtbStPVkhJTDoLnZTy5sejQSPyjfNMy6e4=
github.com/meilisearch/meilisearch-go v0.36.3/go.mod h1:hWcR0MuWLSzHfbz9GGzIr3s9rnXLm1jqkmHkJPbUSvM=
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
@ -354,6 +392,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
@ -480,12 +520,12 @@ github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJ
github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc=
github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA=
github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4=
github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4=
go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk=
go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04=

10231
server/core/internal/_mock/meilisearch_index_manager.go
View File

File diff suppressed because it is too large Load Diff

5186
server/core/internal/_mock/meilisearch_service_manager.go
View File

File diff suppressed because it is too large Load Diff

47
server/core/internal/assistant/manager.go
View File

@ -28,7 +28,6 @@ import (
"github.com/oxynote/oxynote/server/core/internal/assistant/persist"
"github.com/oxynote/oxynote/server/core/internal/assistant/protocol"
"github.com/oxynote/oxynote/server/core/internal/assistant/tools"
"github.com/oxynote/oxynote/server/core/internal/search"
"github.com/oxynote/oxynote/server/core/pkg/errutil"
"github.com/oxynote/oxynote/server/core/pkg/memkit"
"github.com/oxynote/oxynote/server/core/pkg/metricutil"
@ -46,17 +45,17 @@ const _sessionExpiration = time.Hour * 24 * 7
// Manager holds dependencies shared across assistant sessions.
type Manager struct {
log *slog.Logger
db tools.DB
search tools.Searcher
jobs *search.Jobs
runners tools.DataSourceRunners
model model.ToolCallingChatModel
summary model.ToolCallingChatModel
applier tools.EditApplier
tree tools.TreeNotifier
tags tools.TagNotifier
hooks tools.HookNotifier
log *slog.Logger
db tools.DB
search tools.Searcher
searchTrigger tools.SearchTrigger
runners tools.DataSourceRunners
model model.ToolCallingChatModel
summary model.ToolCallingChatModel
applier tools.EditApplier
tree tools.TreeNotifier
tags tools.TagNotifier
hooks tools.HookNotifier
githubMan *github.Manager
webchangeClient *webchange.Client
@ -88,8 +87,8 @@ type Manager struct {
// the provider package from the operator's configuration; summaryModel
// backs context summarisation and may be the same model. The editClient
// is the edit pipe to the Node hocuspocus service; the search client
// backs the search_documents tool and searchJobs is the queue document
// writes announce themselves to; providerName labels token metrics so
// backs the search_documents tool and searchTrigger runs the search-job
// worker once a document write has committed; providerName labels token metrics so
// usage stays readable across a provider change; githubMan and
// webchangeClient are the integrations the hook tools create watchers
// through; the tree notifier broadcasts sidebar refresh events after
@ -105,7 +104,7 @@ func NewManager(
fc metricutil.Factory,
editClient tools.EditApplier,
searcher tools.Searcher,
searchJobs *search.Jobs,
searchTrigger tools.SearchTrigger,
runners tools.DataSourceRunners,
githubMan *github.Manager,
webchangeClient *webchange.Client,
@ -137,14 +136,14 @@ func NewManager(
}
m := &Manager{
log: log,
db: db,
search: searcher,
jobs: searchJobs,
runners: runners,
model: chatModel,
summary: summaryModel,
applier: editClient,
log: log,
db: db,
search: searcher,
searchTrigger: searchTrigger,
runners: runners,
model: chatModel,
summary: summaryModel,
applier: editClient,
githubMan: githubMan,
webchangeClient: webchangeClient,
@ -217,7 +216,7 @@ func (m *Manager) ToolSet(orgID, userID string) *tools.Set {
m.log,
m.db,
m.search,
m.jobs,
m.searchTrigger,
m.runners,
m.githubMan,
m.webchangeClient,

36
server/core/internal/assistant/manager_test.go
View File

@ -18,7 +18,6 @@ import (
persistMock "github.com/oxynote/oxynote/server/core/internal/assistant/persist/_mock"
protocolMock "github.com/oxynote/oxynote/server/core/internal/assistant/protocol/_mock"
toolsMock "github.com/oxynote/oxynote/server/core/internal/assistant/tools/_mock"
"github.com/oxynote/oxynote/server/core/internal/search"
"github.com/oxynote/oxynote/server/core/pkg/errutil"
"github.com/oxynote/oxynote/server/core/pkg/memkit"
"github.com/oxynote/oxynote/server/core/pkg/metricutil"
@ -31,7 +30,7 @@ import (
)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
goleak.VerifyTestMain(m, testutil.IgnoreBleveWorkers())
}
// discardLog returns a logger that writes nowhere.
@ -39,6 +38,11 @@ func discardLog() *slog.Logger {
return slog.New(slog.DiscardHandler)
}
// stubSearchTrigger satisfies the tools' SearchTrigger with a no-op.
type stubSearchTrigger struct{}
func (*stubSearchTrigger) Trigger() {}
// stubChatModel builds a chat model that answers with the given
// messages in order. The last one repeats once the script runs out, so
// a test that under-specifies gets a terminating answer rather than a
@ -186,20 +190,18 @@ func testManagerStores() (*Manager, *stores) {
log := discardLog()
m := &Manager{
log: log,
db: &toolsMock.DB{},
search: &toolsMock.Searcher{
ConfiguredFunc: func() bool { return true },
},
jobs: search.NewJobs(true),
applier: &toolsMock.EditApplier{},
history: persist.NewHistory(log, st.history),
checkpoints: persist.NewCheckpoints(log, st.blobs),
pendings: persist.NewPendings(log, st.pendings),
offload: persist.NewOffload(st.blobs),
metrics: newMetrics(metricutil.NewFactory("test", prometheus.NewRegistry()), "claude"),
model: stubChatModel(),
summary: stubChatModel(),
log: log,
db: &toolsMock.DB{},
search: &toolsMock.Searcher{},
searchTrigger: &stubSearchTrigger{},
applier: &toolsMock.EditApplier{},
history: persist.NewHistory(log, st.history),
checkpoints: persist.NewCheckpoints(log, st.blobs),
pendings: persist.NewPendings(log, st.pendings),
offload: persist.NewOffload(st.blobs),
metrics: newMetrics(metricutil.NewFactory("test", prometheus.NewRegistry()), "claude"),
model: stubChatModel(),
summary: stubChatModel(),
}
m.turns.m = make(map[string]struct{})
@ -238,7 +240,7 @@ func Test_NewManager(t *testing.T) {
fc := metricutil.NewFactory("test", prometheus.NewRegistry())
gh := &github.Manager{}
wc := &webchange.Client{}
m := NewManager(discardLog(), nil, c.Pool, nil, nil, fc, nil, nil, search.NewJobs(false), nil, gh, wc, "claude")
m := NewManager(discardLog(), nil, c.Pool, nil, nil, fc, nil, nil, &stubSearchTrigger{}, nil, gh, wc, "claude")
require.NotNil(t, m)
assert.NotNil(t, m.log)

2
server/core/internal/assistant/middleware/observer_test.go
View File

@ -19,7 +19,7 @@ import (
)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
goleak.VerifyTestMain(m, testutil.IgnoreBleveWorkers())
}
func Test_NewObserver(t *testing.T) {

67
server/core/internal/assistant/tools/0mock_search_trigger_test.go Normal file
View File

@ -0,0 +1,67 @@
// Code generated by moq; DO NOT EDIT.
// github.com/matryer/moq
package tools
import (
"sync"
)
// Ensure, that SearchTriggerMock does implement SearchTrigger.
// If this is not the case, regenerate this file with moq.
var _ SearchTrigger = &SearchTriggerMock{}
// SearchTriggerMock is a mock implementation of SearchTrigger.
//
// func TestSomethingThatUsesSearchTrigger(t *testing.T) {
//
// // make and configure a mocked SearchTrigger
// mockedSearchTrigger := &SearchTriggerMock{
// TriggerFunc: func() {
// panic("mock out the Trigger method")
// },
// }
//
// // use mockedSearchTrigger in code that requires SearchTrigger
// // and then make assertions.
//
// }
type SearchTriggerMock struct {
// TriggerFunc mocks the Trigger method.
TriggerFunc func()
// calls tracks calls to the methods.
calls struct {
// Trigger holds details about calls to the Trigger method.
Trigger []struct {
}
}
lockTrigger sync.RWMutex
}
// Trigger calls TriggerFunc.
func (mock *SearchTriggerMock) Trigger() {
callInfo := struct {
}{}
mock.lockTrigger.Lock()
mock.calls.Trigger = append(mock.calls.Trigger, callInfo)
mock.lockTrigger.Unlock()
if mock.TriggerFunc == nil {
return
}
mock.TriggerFunc()
}
// TriggerCalls gets all the calls that were made to Trigger.
// Check the length with:
//
// len(mockedSearchTrigger.TriggerCalls())
func (mock *SearchTriggerMock) TriggerCalls() []struct {
} {
var calls []struct {
}
mock.lockTrigger.RLock()
calls = mock.calls.Trigger
mock.lockTrigger.RUnlock()
return calls
}

37
server/core/internal/assistant/tools/0mock_searcher_test.go
View File

@ -30,17 +30,11 @@ var _ Searcher = &SearcherMock{}
//
// }
type SearcherMock struct {
// ConfiguredFunc mocks the Configured method.
ConfiguredFunc func() bool
// SearchDocumentBlocksFunc mocks the SearchDocumentBlocks method.
SearchDocumentBlocksFunc func(ctx context.Context, organizationID string, query string, limit int) ([]search.Block, error)
// calls tracks calls to the methods.
calls struct {
// Configured holds details about calls to the Configured method.
Configured []struct {
}
// SearchDocumentBlocks holds details about calls to the SearchDocumentBlocks method.
SearchDocumentBlocks []struct {
// Ctx is the ctx argument value.
@ -53,7 +47,6 @@ type SearcherMock struct {
Limit int
}
}
lockConfigured sync.RWMutex
lockSearchDocumentBlocks sync.RWMutex
}
@ -104,33 +97,3 @@ func (mock *SearcherMock) SearchDocumentBlocksCalls() []struct {
mock.lockSearchDocumentBlocks.RUnlock()
return calls
}
// Configured calls ConfiguredFunc.
func (mock *SearcherMock) Configured() bool {
callInfo := struct {
}{}
mock.lockConfigured.Lock()
mock.calls.Configured = append(mock.calls.Configured, callInfo)
mock.lockConfigured.Unlock()
if mock.ConfiguredFunc == nil {
var (
bOut bool
)
return bOut
}
return mock.ConfiguredFunc()
}
// ConfiguredCalls gets all the calls that were made to Configured.
// Check the length with:
//
// len(mockedSearcher.ConfiguredCalls())
func (mock *SearcherMock) ConfiguredCalls() []struct {
} {
var calls []struct {
}
mock.lockConfigured.RLock()
calls = mock.calls.Configured
mock.lockConfigured.RUnlock()
return calls
}

60
server/core/internal/assistant/tools/0mock_tx_test.go
View File

@ -28,8 +28,8 @@ var _ Tx = &TxMock{}
// InsertDocumentFunc: func(ctx context.Context, doc document.Document) error {
// panic("mock out the InsertDocument method")
// },
// InsertDocumentSearchJobFunc: func(ctx context.Context, diff search.BlocksDifference) error {
// panic("mock out the InsertDocumentSearchJob method")
// InsertSearchJobFunc: func(ctx context.Context, job search.Job) error {
// panic("mock out the InsertSearchJob method")
// },
// RollbackFunc: func() error {
// panic("mock out the Rollback method")
@ -53,8 +53,8 @@ type TxMock struct {
// InsertDocumentFunc mocks the InsertDocument method.
InsertDocumentFunc func(ctx context.Context, doc document.Document) error
// InsertDocumentSearchJobFunc mocks the InsertDocumentSearchJob method.
InsertDocumentSearchJobFunc func(ctx context.Context, diff search.BlocksDifference) error
// InsertSearchJobFunc mocks the InsertSearchJob method.
InsertSearchJobFunc func(ctx context.Context, job search.Job) error
// RollbackFunc mocks the Rollback method.
RollbackFunc func() error
@ -83,12 +83,12 @@ type TxMock struct {
// Doc is the doc argument value.
Doc document.Document
}
// InsertDocumentSearchJob holds details about calls to the InsertDocumentSearchJob method.
InsertDocumentSearchJob []struct {
// InsertSearchJob holds details about calls to the InsertSearchJob method.
InsertSearchJob []struct {
// Ctx is the ctx argument value.
Ctx context.Context
// Diff is the diff argument value.
Diff search.BlocksDifference
// Job is the job argument value.
Job search.Job
}
// Rollback holds details about calls to the Rollback method.
Rollback []struct {
@ -108,7 +108,7 @@ type TxMock struct {
lockCommit sync.RWMutex
lockDeleteDocument sync.RWMutex
lockInsertDocument sync.RWMutex
lockInsertDocumentSearchJob sync.RWMutex
lockInsertSearchJob sync.RWMutex
lockRollback sync.RWMutex
lockUpsertDocumentMaintainers sync.RWMutex
}
@ -182,42 +182,42 @@ func (mock *TxMock) InsertDocumentCalls() []struct {
return calls
}
// InsertDocumentSearchJob calls InsertDocumentSearchJobFunc.
func (mock *TxMock) InsertDocumentSearchJob(ctx context.Context, diff search.BlocksDifference) error {
// InsertSearchJob calls InsertSearchJobFunc.
func (mock *TxMock) InsertSearchJob(ctx context.Context, job search.Job) error {
callInfo := struct {
Ctx context.Context
Diff search.BlocksDifference
Ctx context.Context
Job search.Job
}{
Ctx: ctx,
Diff: diff,
Ctx: ctx,
Job: job,
}
mock.lockInsertDocumentSearchJob.Lock()
mock.calls.InsertDocumentSearchJob = append(mock.calls.InsertDocumentSearchJob, callInfo)
mock.lockInsertDocumentSearchJob.Unlock()
if mock.InsertDocumentSearchJobFunc == nil {
mock.lockInsertSearchJob.Lock()
mock.calls.InsertSearchJob = append(mock.calls.InsertSearchJob, callInfo)
mock.lockInsertSearchJob.Unlock()
if mock.InsertSearchJobFunc == nil {
var (
errOut error
)
return errOut
}
return mock.InsertDocumentSearchJobFunc(ctx, diff)
return mock.InsertSearchJobFunc(ctx, job)
}
// InsertDocumentSearchJobCalls gets all the calls that were made to InsertDocumentSearchJob.
// InsertSearchJobCalls gets all the calls that were made to InsertSearchJob.
// Check the length with:
//
// len(mockedTx.InsertDocumentSearchJobCalls())
func (mock *TxMock) InsertDocumentSearchJobCalls() []struct {
Ctx context.Context
Diff search.BlocksDifference
// len(mockedTx.InsertSearchJobCalls())
func (mock *TxMock) InsertSearchJobCalls() []struct {
Ctx context.Context
Job search.Job
} {
var calls []struct {
Ctx context.Context
Diff search.BlocksDifference
Ctx context.Context
Job search.Job
}
mock.lockInsertDocumentSearchJob.RLock()
calls = mock.calls.InsertDocumentSearchJob
mock.lockInsertDocumentSearchJob.RUnlock()
mock.lockInsertSearchJob.RLock()
calls = mock.calls.InsertSearchJob
mock.lockInsertSearchJob.RUnlock()
return calls
}

50
server/core/internal/assistant/tools/_mock/searcher.go
View File

@ -5,10 +5,9 @@ package mock
import (
"context"
"sync"
"github.com/oxynote/oxynote/server/core/internal/assistant/tools"
"github.com/oxynote/oxynote/server/core/internal/search"
"sync"
)
// Ensure, that Searcher does implement tools.Searcher.
@ -19,29 +18,23 @@ var _ tools.Searcher = &Searcher{}
//
// func TestSomethingThatUsesSearcher(t *testing.T) {
//
// // make and configure a mocked tools.Searcher
// // make and configure a mocked Searcher
// mockedSearcher := &Searcher{
// SearchDocumentBlocksFunc: func(ctx context.Context, organizationID string, query string, limit int) ([]search.Block, error) {
// panic("mock out the SearchDocumentBlocks method")
// },
// }
//
// // use mockedSearcher in code that requires tools.Searcher
// // use mockedSearcher in code that requires Searcher
// // and then make assertions.
//
// }
type Searcher struct {
// ConfiguredFunc mocks the Configured method.
ConfiguredFunc func() bool
// SearchDocumentBlocksFunc mocks the SearchDocumentBlocks method.
SearchDocumentBlocksFunc func(ctx context.Context, organizationID string, query string, limit int) ([]search.Block, error)
// calls tracks calls to the methods.
calls struct {
// Configured holds details about calls to the Configured method.
Configured []struct {
}
// SearchDocumentBlocks holds details about calls to the SearchDocumentBlocks method.
SearchDocumentBlocks []struct {
// Ctx is the ctx argument value.
@ -54,7 +47,6 @@ type Searcher struct {
Limit int
}
}
lockConfigured sync.RWMutex
lockSearchDocumentBlocks sync.RWMutex
}
@ -76,10 +68,10 @@ func (mock *Searcher) SearchDocumentBlocks(ctx context.Context, organizationID s
mock.lockSearchDocumentBlocks.Unlock()
if mock.SearchDocumentBlocksFunc == nil {
var (
blocksOut []search.Block
errOut error
out0 []search.Block
errOut error
)
return blocksOut, errOut
return out0, errOut
}
return mock.SearchDocumentBlocksFunc(ctx, organizationID, query, limit)
}
@ -105,33 +97,3 @@ func (mock *Searcher) SearchDocumentBlocksCalls() []struct {
mock.lockSearchDocumentBlocks.RUnlock()
return calls
}
// Configured calls ConfiguredFunc.
func (mock *Searcher) Configured() bool {
callInfo := struct {
}{}
mock.lockConfigured.Lock()
mock.calls.Configured = append(mock.calls.Configured, callInfo)
mock.lockConfigured.Unlock()
if mock.ConfiguredFunc == nil {
var (
bOut bool
)
return bOut
}
return mock.ConfiguredFunc()
}
// ConfiguredCalls gets all the calls that were made to Configured.
// Check the length with:
//
// len(mockedSearcher.ConfiguredCalls())
func (mock *Searcher) ConfiguredCalls() []struct {
} {
var calls []struct {
}
mock.lockConfigured.RLock()
calls = mock.calls.Configured
mock.lockConfigured.RUnlock()
return calls
}

4
server/core/internal/assistant/tools/document_test.go
View File

@ -29,7 +29,7 @@ func stubCreateDB(checkErr error) *DBMock {
UpsertDocumentMaintainersFunc: func(context.Context, xid.ID, string, []string) error {
return nil
},
InsertDocumentSearchJobFunc: func(context.Context, search.BlocksDifference) error {
InsertSearchJobFunc: func(context.Context, search.Job) error {
return nil
},
}))
@ -601,7 +601,7 @@ func Test_deleteDocument_Execute(t *testing.T) {
DeleteDocumentFunc: func(_ context.Context, id xid.ID, _ string) ([]xid.ID, error) {
return []xid.ID{id}, deleteErr
},
InsertDocumentSearchJobFunc: func(context.Context, search.BlocksDifference) error {
InsertSearchJobFunc: func(context.Context, search.Job) error {
return nil
},
}))

50
server/core/internal/assistant/tools/input.go
View File

@ -59,10 +59,9 @@ type Deps struct {
// search is the full-text index behind search_documents.
search Searcher
// jobs is the way into the search-job queue: it decides whether a
// deployment indexes at all, so writes queue through it instead of
// hitting the database directly.
jobs *search.Jobs
// searchTrigger runs the search-job worker once a write that queued a
// job has committed.
searchTrigger SearchTrigger
// runners hands out the runner a data-source tool reads through.
runners DataSourceRunners
@ -112,7 +111,7 @@ func NewDeps(
log *slog.Logger,
db DB,
searcher Searcher,
jobs *search.Jobs,
searchTrigger SearchTrigger,
runners DataSourceRunners,
githubMan *github.Manager,
webchangeClient *webchange.Client,
@ -131,7 +130,7 @@ func NewDeps(
),
db: db,
search: searcher,
jobs: jobs,
searchTrigger: searchTrigger,
runners: runners,
githubMan: githubMan,
webchangeClient: webchangeClient,
@ -451,7 +450,7 @@ func (i *input) CreateDocument(doc document.Document) error {
// without this the document is invisible to search until someone
// edits it, since only the persist path queues a job.
if err := i.jobs.Enqueue(i.ctx, tx, search.BlocksDiff(nil, doc.Search())); err != nil {
if err := tx.InsertSearchJob(i.ctx, search.BranchScope(i.orgID, doc.ID, doc.BranchID)); err != nil {
return fmt.Errorf("insert search job: %w", err)
}
@ -459,6 +458,7 @@ func (i *input) CreateDocument(doc document.Document) error {
return fmt.Errorf("commit: %w", err)
}
i.searchTrigger.Trigger()
i.recordTouched(doc.ID, doc.BranchID)
return nil
@ -511,13 +511,19 @@ func (i *input) DeleteDocument(id xid.ID) error {
return ErrUnknownDocument
}
if err := i.jobs.Enqueue(i.ctx, tx, search.BlocksDifference{
RemovedDocuments: ids,
}); err != nil {
return fmt.Errorf("insert search job: %w", err)
for _, id := range ids {
if err := tx.InsertSearchJob(i.ctx, search.DocumentScope(i.orgID, id)); err != nil {
return fmt.Errorf("insert search job: %w", err)
}
}
return tx.Commit()
if err := tx.Commit(); err != nil {
return fmt.Errorf("commit: %w", err)
}
i.searchTrigger.Trigger()
return nil
}
// MoveDocument re-parents the document doc describes.
@ -1317,9 +1323,9 @@ type Tx interface {
// document's maintainer set. Used by create_document.
UpsertDocumentMaintainers(ctx context.Context, documentID xid.ID, organizationID string, maintainerIDs []string) error
// InsertDocumentSearchJob should queue the search index update for a
// document. Used by create_document and delete_document.
InsertDocumentSearchJob(ctx context.Context, diff search.BlocksDifference) error
// InsertSearchJob should queue the search job's scope. Used by
// create_document and delete_document.
InsertSearchJob(ctx context.Context, job search.Job) error
// DeleteDocument should remove a document and report the ids of the
// document and of every cascade-deleted descendant. Used by
@ -1327,15 +1333,19 @@ type Tx interface {
DeleteDocument(ctx context.Context, id xid.ID, organizationID string) ([]xid.ID, error)
}
// SearchTrigger runs the search-job worker once a job has committed.
//
//go:generate ../../../scripts/codegen/mock -t internal SearchTrigger search_trigger
type SearchTrigger interface {
// Trigger should run a search-job pass right away.
Trigger()
}
// Searcher is the full-text search surface search_documents uses.
// The document/search Meilisearch client satisfies it.
// The search index satisfies it.
//
//go:generate ../../../scripts/codegen/mock -t both Searcher searcher
type Searcher interface {
// Configured should report whether search is configured on this
// deployment.
Configured() bool
// SearchDocumentBlocks should return blocks whose text matches the
// query, scoped to the organization and capped at limit hits.
SearchDocumentBlocks(ctx context.Context, organizationID, query string, limit int) ([]search.Block, error)

30
server/core/internal/assistant/tools/input_test.go
View File

@ -34,7 +34,7 @@ import (
)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
goleak.VerifyTestMain(m, testutil.IgnoreBleveWorkers())
}
// _testDocID is the document id the tool tests address.
@ -72,12 +72,10 @@ func testDeps(db *DBMock, applier *EditApplierMock, tree *TreeNotifierMock) *Dep
}
d := &Deps{
log: discardLog(),
db: db,
search: &SearcherMock{
ConfiguredFunc: func() bool { return true },
},
jobs: search.NewJobs(true),
log: discardLog(),
db: db,
search: &SearcherMock{},
searchTrigger: &SearchTriggerMock{},
runners: &DataSourceRunnersMock{
RunnerFunc: func(datasource.DataSource) datasource.Runner {
return &datasourceMock.Runner{}
@ -405,15 +403,15 @@ func Test_NewDeps(t *testing.T) {
RunnerFunc: func(datasource.DataSource) datasource.Runner { return runner },
}
jobs := search.NewJobs(true)
trigger := &SearchTriggerMock{}
d := NewDeps(discardLog(), db, searcher, jobs, runners, gh, wc, applier, tree, tags, hooks, offload, "org", "user")
d := NewDeps(discardLog(), db, searcher, trigger, runners, gh, wc, applier, tree, tags, hooks, offload, "org", "user")
require.NotNil(t, d)
assert.NotNil(t, d.log)
assert.Same(t, db, d.db)
assert.Same(t, searcher, d.search)
assert.Same(t, jobs, d.jobs)
assert.Same(t, trigger, d.searchTrigger)
assert.Same(t, runners, d.runners)
assert.Same(t, gh, d.githubMan)
assert.Same(t, wc, d.webchangeClient)
@ -520,7 +518,7 @@ func Test_input_CreateDocument(t *testing.T) {
UpsertDocumentMaintainersFunc: func(context.Context, xid.ID, string, []string) error {
return upsertErr
},
InsertDocumentSearchJobFunc: func(context.Context, search.BlocksDifference) error {
InsertSearchJobFunc: func(context.Context, search.Job) error {
return jobErr
},
CommitFunc: func() error { return commitErr },
@ -631,7 +629,7 @@ func Test_input_DeleteDocument(t *testing.T) {
DeleteDocumentFunc: func(context.Context, xid.ID, string) ([]xid.ID, error) {
return ids, deleteErr
},
InsertDocumentSearchJobFunc: func(context.Context, search.BlocksDifference) error {
InsertSearchJobFunc: func(context.Context, search.Job) error {
return jobErr
},
CommitFunc: func() error { return commitErr },
@ -681,7 +679,8 @@ func Test_input_DeleteDocument(t *testing.T) {
"Delete and search removal land together": func() tcase {
tx := stubTx([]xid.ID{docID, xid.New()}, nil, nil, nil)
return tcase{DB: stubDB(tx, nil), Tx: tx, Commits: 1, Jobs: 1}
// one job per cascade-deleted document.
return tcase{DB: stubDB(tx, nil), Tx: tx, Commits: 1, Jobs: 2}
}(),
}
@ -706,11 +705,12 @@ func Test_input_DeleteDocument(t *testing.T) {
assert.Len(t, c.Tx.CommitCalls(), c.Commits)
assert.Len(t, c.Tx.RollbackCalls(), 1)
ff := c.Tx.InsertDocumentSearchJobCalls()
ff := c.Tx.InsertSearchJobCalls()
require.Len(t, ff, c.Jobs)
if c.Jobs != 0 {
assert.NotEmpty(t, ff[0].Diff.RemovedDocuments)
assert.True(t, ff[0].Job.DocumentID.Valid)
assert.False(t, ff[0].Job.BranchID.Valid)
}
})
}

13
server/core/internal/assistant/tools/tools.go
View File

@ -106,16 +106,7 @@ func New(deps *Deps) *Set {
readBlock{},
listTags{},
listHooks{},
}
// a deployment without search has no index behind search_documents,
// so the tool is left out of the registry entirely — neither the
// model nor the MCP surface is offered a tool that cannot work.
if deps.search.Configured() {
all = append(all, searchDocuments{})
}
all = append(all,
searchDocuments{},
listDataSources{},
getPrometheusMetadata{},
listPrometheusLabelNames{},
@ -147,7 +138,7 @@ func New(deps *Deps) *Set {
deleteHook{},
readToolOutput{},
)
}
s := &Set{tools: make(map[Name]registryTool, len(all))}

20
server/core/internal/assistant/tools/tools_test.go
View File

@ -121,26 +121,6 @@ func Test_New(t *testing.T) {
}
}
func Test_New_withoutSearch(t *testing.T) {
t.Parallel()
d := testDeps(nil, nil, nil)
d.search = &SearcherMock{}
s := New(d)
// without search there is no index behind search_documents, so the
// registry offers it to no surface at all; everything else stays.
require.Len(t, s.tools, len(allToolNames())-1)
_, ok := s.tools[NameSearchDocuments]
assert.False(t, ok)
for _, e := range s.Entries() {
assert.NotEqual(t, string(NameSearchDocuments), e.Name)
}
}
func Test_Set_Tools(t *testing.T) {
t.Parallel()

23
server/core/internal/db/document_branch.go
View File

@ -187,6 +187,29 @@ func (a *agent) FetchDocumentUnsafeByBranchID(ctx context.Context, branchID xid.
return doc, nil
}
// FetchDocumentBranchesAfter fetches up to limit branches whose id sorts
// after the given one, joined against their document, in id order and
// across every organization.
func (a *agent) FetchDocumentBranchesAfter(ctx context.Context, after xid.ID, limit int) ([]document.Document, error) {
// a zero id is compared as its string form, since its driver value
// would be a null the comparison cannot take.
q, args := a.selectDocumentBranch(a.builder.Select()).
Where(sq.Gt{
"db.id": after.String(),
}).
OrderBy("db.id ASC").
Limit(uint64(limit)).
MustSql()
docs := []document.Document{}
if err := sqlx.SelectContext(ctx, a.sql, &docs, q, args...); err != nil {
return nil, err
}
return docs, nil
}
// selectDocumentBranch prepares a select statement joining a document against
// one of its branches. The organization scope is left to the caller, since
// the unsafe variants exist precisely to go without it.

53
server/core/internal/db/document_branch_test.go
View File

@ -600,6 +600,59 @@ func Test_agent_FetchDocumentUnsafeByBranchID(t *testing.T) {
testutil.AssertFilterEqual(t, branch, res)
}
func Test_agent_FetchDocumentBranchesAfter(t *testing.T) {
db := prepTempDB(t)
// error - cancelled context
ctx, cancel := context.WithCancel(context.Background())
cancel()
res, err := db.FetchDocumentBranchesAfter(ctx, xid.ID{}, 10)
require.Error(t, err)
assert.Nil(t, res)
// success - no branches
res, err = db.FetchDocumentBranchesAfter(context.Background(), xid.ID{}, 10)
require.NoError(t, err)
assert.Empty(t, res)
// success - every branch across organizations, in id order. Each
// prepared document comes with its own default branch, so two documents
// and three extra branches make five.
branches := prepDocumentBranches(t, db, 3, func(i int, doc *document.Document) {
if i == 2 {
other := prepDocuments(t, db, 1, nil)[0]
doc.ID = other.ID
doc.OrganizationID = other.OrganizationID
}
})
all, err := db.FetchDocumentBranchesAfter(context.Background(), xid.ID{}, 10)
require.NoError(t, err)
require.Len(t, all, 5)
ids := make([]string, 0, len(all))
for _, doc := range all {
ids = append(ids, doc.BranchID.String())
}
assert.IsIncreasing(t, ids)
for _, branch := range branches {
assert.Contains(t, ids, branch.BranchID.String())
}
// success - a page, then the rest after the page's last id
res, err = db.FetchDocumentBranchesAfter(context.Background(), xid.ID{}, 2)
require.NoError(t, err)
testutil.AssertFilterEqual(t, all[:2], res, null.Value[xid.ID]{})
res, err = db.FetchDocumentBranchesAfter(context.Background(), all[1].BranchID, 10)
require.NoError(t, err)
testutil.AssertFilterEqual(t, all[2:], res, null.Value[xid.ID]{})
}
// prepHistoryEntries writes count entries of the document's branch, one per
// aggregation bucket so each lands as its own entry, oldest first.
func prepHistoryEntries(t *testing.T, db *DB, doc *document.Document, count int) []document.HistoryEntry {

65
server/core/internal/db/document_search_job.go
View File

@ -1,65 +0,0 @@
package db
import (
"context"
sq "github.com/Masterminds/squirrel"
"github.com/jmoiron/sqlx"
"github.com/oxynote/oxynote/server/core/internal/search"
)
// InsertDocumentSearchJob inserts a new document search job into the database.
func (a *agent) InsertDocumentSearchJob(ctx context.Context, blockDiff search.BlocksDifference) error {
return a.insertDocumentSearchJob(ctx, a.sql, blockDiff)
}
// insertDocumentSearchJob inserts a search job through the given executor so
// an agent method that already opened a transaction can queue one inside it.
func (a *agent) insertDocumentSearchJob(
ctx context.Context,
ex sqlx.ExecerContext,
blockDiff search.BlocksDifference,
) error {
q, args := a.builder.Insert("document_search_jobs").
SetMap(map[string]any{
"block_diff": blockDiff,
}).MustSql()
_, err := ex.ExecContext(ctx, q, args...)
return err
}
// FetchDocumentSearchJobs retrieves a batch of document search jobs with IDs
// greater than the given offset ID.
func (a *agent) FetchDocumentSearchJobs(ctx context.Context, offsetID, limit int64) ([]search.DocumentSearchJob, error) {
q, args := a.builder.Select(
"id",
"block_diff",
).From("document_search_jobs").
Where(sq.Gt{"id": offsetID}).
OrderBy("id ASC").
Limit(uint64(limit)).
MustSql()
var jobs []search.DocumentSearchJob
err := sqlx.SelectContext(ctx, a.sql, &jobs, q, args...)
if err != nil {
return nil, err
}
return jobs, nil
}
// DeleteDocumentSearchJob deletes a document search job from the database.
func (a *agent) DeleteDocumentSearchJob(ctx context.Context, id int64) error {
q, args := a.builder.Delete("document_search_jobs").
Where(sq.Eq{
"id": id,
}).MustSql()
_, err := a.sql.ExecContext(ctx, q, args...)
return err
}

221
server/core/internal/db/document_search_job_test.go
View File

@ -1,221 +0,0 @@
package db
import (
"context"
"database/sql"
"strconv"
"testing"
sq "github.com/Masterminds/squirrel"
"github.com/oxynote/oxynote/server/core/internal/search"
"github.com/oxynote/oxynote/server/core/pkg/testutil"
"github.com/rs/xid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func prepDocumentSearchJobs(t *testing.T, db *DB, count int, fn func(int, *search.DocumentSearchJob)) []search.DocumentSearchJob {
t.Helper()
res := make([]search.DocumentSearchJob, count)
for i := range count {
job := search.DocumentSearchJob{
BlockDiff: search.BlocksDifference{
Updated: []search.Block{
{
ID: "block-" + strconv.Itoa(i),
OrganizationID: "org-" + strconv.Itoa(i),
DocumentID: xid.New(),
Type: "paragraph",
Text: "Updated block " + strconv.Itoa(i),
},
},
},
}
if fn != nil {
fn(i, &job)
}
// the id column is a serial; let the database assign it so
// the sequence stays usable for production inserts.
q, args := db.builder.Insert("document_search_jobs").
SetMap(map[string]any{
"block_diff": job.BlockDiff,
}).
Suffix("RETURNING id").
MustSql()
err := db.sql.Get(&job.ID, q, args...)
require.NoError(t, err)
res[i] = job
}
return res
}
func Test_agent_InsertDocumentSearchJob(t *testing.T) {
type tcase struct {
CancelledContext bool
BlockDiff search.BlocksDifference
Err error
}
cc := map[string]func(*testing.T, *DB) tcase{
"Cancelled context": func(_ *testing.T, _ *DB) tcase {
return tcase{
CancelledContext: true,
Err: assert.AnError,
}
},
"Successful insert": func(_ *testing.T, _ *DB) tcase {
return tcase{
BlockDiff: search.BlocksDifference{
Added: []search.Block{
{
ID: "block-added",
OrganizationID: "org-added",
DocumentID: xid.New(),
Type: "heading",
Text: "Added block",
},
},
Removed: []search.Block{
{
ID: "block-removed",
OrganizationID: "org-removed",
DocumentID: xid.New(),
Type: "paragraph",
Text: "Removed block",
},
},
},
}
},
}
for cn, cfn := range cc {
t.Run(cn, func(t *testing.T) {
t.Parallel()
db := prepTempDB(t)
c := cfn(t, db)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if c.CancelledContext {
cancel()
}
err := db.InsertDocumentSearchJob(ctx, c.BlockDiff)
testutil.RequireEqualError(t, c.Err, err)
if err != nil {
return
}
var blockDiff search.BlocksDifference
q, args := db.builder.Select("block_diff").
From("document_search_jobs").
MustSql()
err = db.sql.Get(&blockDiff, q, args...)
require.NoError(t, err)
assert.Equal(t, c.BlockDiff, blockDiff)
})
}
}
func Test_agent_FetchDocumentSearchJobs(t *testing.T) {
db := prepTempDB(t)
// error - cancelled context
ctx, cancel := context.WithCancel(context.Background())
cancel()
res, err := db.FetchDocumentSearchJobs(ctx, 0, 10)
require.Error(t, err)
assert.Nil(t, res)
// success - no jobs
res, err = db.FetchDocumentSearchJobs(context.Background(), 0, 10)
require.NoError(t, err)
assert.Empty(t, res)
// success - limited batch in id order
jobs := prepDocumentSearchJobs(t, db, 3, nil)
res, err = db.FetchDocumentSearchJobs(context.Background(), 0, 2)
assert.NoError(t, err)
assert.Equal(t, jobs[:2], res)
// success - offset skips earlier jobs
res, err = db.FetchDocumentSearchJobs(context.Background(), jobs[0].ID, 10)
assert.NoError(t, err)
assert.Equal(t, jobs[1:], res)
}
func Test_agent_DeleteDocumentSearchJob(t *testing.T) {
type tcase struct {
CancelledContext bool
ID int64
Err error
}
cc := map[string]func(*testing.T, *DB) tcase{
"Cancelled context": func(t *testing.T, db *DB) tcase {
jobs := prepDocumentSearchJobs(t, db, 1, nil)
return tcase{
CancelledContext: true,
ID: jobs[0].ID,
Err: assert.AnError,
}
},
"Successful delete": func(t *testing.T, db *DB) tcase {
jobs := prepDocumentSearchJobs(t, db, 1, nil)
return tcase{
ID: jobs[0].ID,
}
},
}
for cn, cfn := range cc {
t.Run(cn, func(t *testing.T) {
t.Parallel()
db := prepTempDB(t)
c := cfn(t, db)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if c.CancelledContext {
cancel()
}
err := db.DeleteDocumentSearchJob(ctx, c.ID)
testutil.RequireEqualError(t, c.Err, err)
if err != nil {
return
}
var id int64
q, args := db.builder.Select("id").
From("document_search_jobs").
Where(sq.Eq{
"id": c.ID,
}).MustSql()
err = db.sql.Get(&id, q, args...)
testutil.AssertEqualError(t, sql.ErrNoRows, err)
})
}
}

2
server/core/internal/db/document_test.go
View File

@ -1025,7 +1025,7 @@ func Test_agent_DeleteDocument(t *testing.T) {
// the delete itself queues nothing: the caller owns the
// search-index removal.
jobs, err := db.FetchDocumentSearchJobs(context.Background(), 0, 10)
jobs, err := db.FetchSearchJobs(context.Background(), 0, 10)
require.NoError(t, err)
assert.Empty(t, jobs)

17
server/core/internal/db/migrations/001_initial.sql
View File

@ -420,9 +420,18 @@ CREATE TABLE document_files (
CREATE INDEX document_files_fk_document_id_idx ON document_files (fk_document_id);
CREATE INDEX document_files_fk_organization_id_idx ON document_files (fk_organization_id);
CREATE TABLE document_search_jobs (
id SERIAL PRIMARY KEY,
block_diff JSONB NOT NULL
-- Search-index resync jobs. A job names a scope (an organization, one of
-- its documents, or one branch) whose entries the worker rebuilds from the
-- rows; there are no foreign keys, since a job must outlive the rows it
-- removes from the index. Queuing a pending scope again bumps its version
-- instead of adding a row.
CREATE TABLE search_jobs (
id BIGSERIAL PRIMARY KEY,
version BIGINT NOT NULL DEFAULT 1,
organization_id TEXT NOT NULL,
document_id TEXT,
branch_id TEXT,
UNIQUE NULLS NOT DISTINCT (organization_id, document_id, branch_id)
);
-- Slack / GitHub integrations and the per-org notification queue.
@ -477,7 +486,7 @@ DROP TABLE slack_messages;
DROP TABLE github_installations;
DROP TABLE slack_user_links;
DROP TABLE slack_apps;
DROP TABLE document_search_jobs;
DROP TABLE search_jobs;
DROP TABLE document_files;
DROP TABLE branch_reviewers;
DROP TABLE user_tag_settings;

66
server/core/internal/db/search_job.go Normal file
View File

@ -0,0 +1,66 @@
package db
import (
"context"
sq "github.com/Masterminds/squirrel"
"github.com/jmoiron/sqlx"
"github.com/oxynote/oxynote/server/core/internal/search"
)
// InsertSearchJob queues the job's scope. A scope already pending has its
// version bumped instead, so the worker applies it once more if it was
// mid-application.
func (a *agent) InsertSearchJob(ctx context.Context, job search.Job) error {
q, args := a.builder.Insert("search_jobs").
SetMap(map[string]any{
"organization_id": job.OrganizationID,
"document_id": job.DocumentID,
"branch_id": job.BranchID,
}).
Suffix("ON CONFLICT (organization_id, document_id, branch_id) DO UPDATE SET version = search_jobs.version + 1").
MustSql()
_, err := a.sql.ExecContext(ctx, q, args...)
return err
}
// FetchSearchJobs retrieves a batch of search jobs with IDs greater than the
// given offset ID.
func (a *agent) FetchSearchJobs(ctx context.Context, offsetID, limit int64) ([]search.Job, error) {
q, args := a.builder.Select(
"id",
"version",
"organization_id",
"document_id",
"branch_id",
).From("search_jobs").
Where(sq.Gt{"id": offsetID}).
OrderBy("id ASC").
Limit(uint64(limit)).
MustSql()
jobs := []search.Job{}
err := sqlx.SelectContext(ctx, a.sql, &jobs, q, args...)
if err != nil {
return nil, err
}
return jobs, nil
}
// DeleteSearchJob deletes the job at the given version. A job queued again
// meanwhile carries a newer version and stays.
func (a *agent) DeleteSearchJob(ctx context.Context, id, version int64) error {
q, args := a.builder.Delete("search_jobs").
Where(sq.Eq{
"id": id,
"version": version,
}).MustSql()
_, err := a.sql.ExecContext(ctx, q, args...)
return err
}

302
server/core/internal/db/search_job_test.go Normal file
View File

@ -0,0 +1,302 @@
package db
import (
"context"
"database/sql"
"strconv"
"testing"
sq "github.com/Masterminds/squirrel"
"github.com/guregu/null/v5"
"github.com/oxynote/oxynote/server/core/internal/search"
"github.com/oxynote/oxynote/server/core/pkg/testutil"
"github.com/rs/xid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func prepSearchJobs(t *testing.T, db *DB, count int, fn func(int, *search.Job)) []search.Job {
t.Helper()
res := make([]search.Job, count)
for i := range count {
job := search.BranchScope("org-"+strconv.Itoa(i), xid.New(), xid.New())
job.Version = 1
if fn != nil {
fn(i, &job)
}
// the id column is a serial; let the database assign it so
// the sequence stays usable for production inserts.
q, args := db.builder.Insert("search_jobs").
SetMap(map[string]any{
"version": job.Version,
"organization_id": job.OrganizationID,
"document_id": job.DocumentID,
"branch_id": job.BranchID,
}).
Suffix("RETURNING id").
MustSql()
err := db.sql.Get(&job.ID, q, args...)
require.NoError(t, err)
res[i] = job
}
return res
}
// fetchSearchJobs reads every queued job in id order.
func fetchSearchJobs(t *testing.T, db *DB) []search.Job {
t.Helper()
q, args := db.builder.Select(
"id",
"version",
"organization_id",
"document_id",
"branch_id",
).From("search_jobs").
OrderBy("id ASC").
MustSql()
var jobs []search.Job
require.NoError(t, db.sql.Select(&jobs, q, args...))
return jobs
}
func Test_agent_InsertSearchJob(t *testing.T) {
type tcase struct {
CancelledContext bool
Job search.Job
Result []search.Job
Err error
}
documentID, branchID := xid.New(), xid.New()
cc := map[string]func(*testing.T, *DB) tcase{
"Cancelled context": func(_ *testing.T, _ *DB) tcase {
return tcase{
CancelledContext: true,
Job: search.BranchScope("org-1", documentID, branchID),
Err: assert.AnError,
}
},
"Successful insert of a branch scope": func(_ *testing.T, _ *DB) tcase {
job := search.BranchScope("org-1", documentID, branchID)
job.Version = 1
return tcase{
Job: job,
Result: []search.Job{job},
}
},
"Successful insert of a document scope": func(_ *testing.T, _ *DB) tcase {
job := search.DocumentScope("org-1", documentID)
job.Version = 1
return tcase{
Job: job,
Result: []search.Job{job},
}
},
"Successful insert of an organization scope": func(_ *testing.T, _ *DB) tcase {
job := search.OrganizationScope("org-1")
job.Version = 1
return tcase{
Job: job,
Result: []search.Job{job},
}
},
"Pending scope gets its version bumped": func(t *testing.T, db *DB) tcase {
pending := prepSearchJobs(t, db, 1, func(_ int, job *search.Job) {
*job = search.BranchScope("org-1", documentID, branchID)
job.Version = 1
})[0]
pending.Version = 2
return tcase{
Job: search.BranchScope("org-1", documentID, branchID),
Result: []search.Job{pending},
}
},
"Pending scope with nulls gets its version bumped": func(t *testing.T, db *DB) tcase {
pending := prepSearchJobs(t, db, 1, func(_ int, job *search.Job) {
*job = search.OrganizationScope("org-1")
job.Version = 1
})[0]
pending.Version = 2
return tcase{
Job: search.OrganizationScope("org-1"),
Result: []search.Job{pending},
}
},
"Different scope of the same organization is its own job": func(t *testing.T, db *DB) tcase {
pending := prepSearchJobs(t, db, 1, func(_ int, job *search.Job) {
*job = search.OrganizationScope("org-1")
job.Version = 1
})[0]
job := search.DocumentScope("org-1", documentID)
job.Version = 1
return tcase{
Job: job,
Result: []search.Job{pending, job},
}
},
}
for cn, cfn := range cc {
t.Run(cn, func(t *testing.T) {
t.Parallel()
db := prepTempDB(t)
c := cfn(t, db)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if c.CancelledContext {
cancel()
}
err := db.InsertSearchJob(ctx, c.Job)
testutil.RequireEqualError(t, c.Err, err)
if err != nil {
return
}
testutil.AssertFilterEqual(t, c.Result, fetchSearchJobs(t, db), int64(0))
})
}
}
func Test_agent_FetchSearchJobs(t *testing.T) {
db := prepTempDB(t)
// error - cancelled context
ctx, cancel := context.WithCancel(context.Background())
cancel()
res, err := db.FetchSearchJobs(ctx, 0, 10)
require.Error(t, err)
assert.Nil(t, res)
// success - no jobs
res, err = db.FetchSearchJobs(context.Background(), 0, 10)
require.NoError(t, err)
assert.Equal(t, []search.Job{}, res)
// success - limited batch in id order, nullable scopes included
jobs := prepSearchJobs(t, db, 3, func(i int, job *search.Job) {
switch i {
case 1:
job.BranchID = null.Value[xid.ID]{}
case 2:
job.DocumentID = null.Value[xid.ID]{}
job.BranchID = null.Value[xid.ID]{}
}
})
res, err = db.FetchSearchJobs(context.Background(), 0, 2)
assert.NoError(t, err)
assert.Equal(t, jobs[:2], res)
// success - offset skips earlier jobs
res, err = db.FetchSearchJobs(context.Background(), jobs[0].ID, 10)
assert.NoError(t, err)
assert.Equal(t, jobs[1:], res)
}
func Test_agent_DeleteSearchJob(t *testing.T) {
type tcase struct {
CancelledContext bool
ID int64
Version int64
Deleted bool
Err error
}
cc := map[string]func(*testing.T, *DB) tcase{
"Cancelled context": func(t *testing.T, db *DB) tcase {
jobs := prepSearchJobs(t, db, 1, nil)
return tcase{
CancelledContext: true,
ID: jobs[0].ID,
Version: jobs[0].Version,
Err: assert.AnError,
}
},
"Newer version keeps the job": func(t *testing.T, db *DB) tcase {
jobs := prepSearchJobs(t, db, 1, func(_ int, job *search.Job) {
job.Version = 2
})
return tcase{
ID: jobs[0].ID,
Version: 1,
Deleted: false,
}
},
"Successful delete": func(t *testing.T, db *DB) tcase {
jobs := prepSearchJobs(t, db, 1, nil)
return tcase{
ID: jobs[0].ID,
Version: jobs[0].Version,
Deleted: true,
}
},
}
for cn, cfn := range cc {
t.Run(cn, func(t *testing.T) {
t.Parallel()
db := prepTempDB(t)
c := cfn(t, db)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if c.CancelledContext {
cancel()
}
err := db.DeleteSearchJob(ctx, c.ID, c.Version)
testutil.RequireEqualError(t, c.Err, err)
if err != nil {
return
}
var id int64
q, args := db.builder.Select("id").
From("search_jobs").
Where(sq.Eq{
"id": c.ID,
}).MustSql()
err = db.sql.Get(&id, q, args...)
if c.Deleted {
testutil.AssertEqualError(t, sql.ErrNoRows, err)
return
}
require.NoError(t, err)
})
}
}

20
server/core/internal/document/document.go
View File

@ -10,7 +10,6 @@ import (
"time"
"github.com/guregu/null/v5"
"github.com/oxynote/oxynote/server/core/internal/search"
"github.com/oxynote/oxynote/server/core/pkg/httpserver"
"github.com/oxynote/oxynote/server/core/pkg/strutil"
"github.com/oxynote/oxynote/server/core/pkg/timeutil"
@ -299,25 +298,6 @@ func (d Document) HistoryEntry() HistoryEntry {
}
}
// Search converts the document's branch to search blocks for indexing.
func (d Document) Search() map[string]search.Block {
scope := search.Scope{
OrganizationID: d.OrganizationID,
DocumentID: d.ID,
BranchID: d.BranchID,
BranchName: d.BranchName,
BranchDefault: d.Default,
}
res := d.Content.Search(scope)
// a special block carrying the document name. The map is keyed by the
// document ID, which no content block shares.
res[d.ID.String()] = scope.Block("docname", "document", d.DocumentName)
return res
}
// HistoryEntry represents one entry in a document's history.
type HistoryEntry struct {
// ID is the unique identifier for the entry.

31
server/core/internal/document/document_test.go
View File

@ -5,7 +5,6 @@ import (
"time"
"github.com/guregu/null/v5"
"github.com/oxynote/oxynote/server/core/internal/search"
"github.com/oxynote/oxynote/server/core/pkg/httpserver"
"github.com/rs/xid"
"github.com/stretchr/testify/assert"
@ -286,36 +285,6 @@ func Test_Document_HistoryEntry(t *testing.T) {
assert.NotEqual(t, entry.ID, other.HistoryEntry().ID)
}
func Test_Document_Search(t *testing.T) {
t.Parallel()
doc := stubDocument()
res := doc.Search()
// the content block plus the synthetic document-name block.
require.Len(t, res, 2)
nameBlock, ok := res[doc.ID.String()]
require.True(t, ok)
assert.Equal(t, search.Block{
ID: doc.BranchID.String() + "-docname",
OrganizationID: "org-1",
DocumentID: doc.ID,
BranchID: doc.BranchID,
BranchName: DefaultBranch,
BranchDefault: true,
Type: "document",
Text: "Runbook",
}, nameBlock)
contentBlock, ok := res["p1"]
require.True(t, ok)
assert.Equal(t, doc.BranchID.String()+"-p1", contentBlock.ID)
assert.Equal(t, doc.BranchID, contentBlock.BranchID)
assert.Equal(t, "hello", contentBlock.Text)
}
func Test_Document_Duplicate(t *testing.T) {
t.Parallel()

57
server/core/internal/document/node.go
View File

@ -5,11 +5,9 @@ import (
"encoding/json"
"errors"
"fmt"
"maps"
"net/url"
"strings"
"github.com/oxynote/oxynote/server/core/internal/search"
"github.com/oxynote/oxynote/server/core/pkg/strutil"
"github.com/rs/xid"
)
@ -77,19 +75,6 @@ func (rb RootBlock) HasBlock(blockID string) bool {
return false
}
// Search transforms RootBlock into a search compatible type.
func (rb RootBlock) Search(scope search.Scope) map[string]search.Block {
res := make(map[string]search.Block)
for _, b := range rb.Content {
content := b.Search(scope)
maps.Insert(res, maps.All(content))
}
return res
}
// Value transforms stopper type into a database entry.
func (rb RootBlock) Value() (driver.Value, error) {
return json.Marshal(rb)
@ -217,48 +202,6 @@ func (b Block) HasBlock(blockID string) bool {
return ok
}
// Search transforms Block into a search compatible type.
func (b Block) Search(scope search.Scope) map[string]search.Block {
res := make(map[string]search.Block)
var text strings.Builder
for _, cb := range b.Content {
// All parent nodes that contain text have a content element
// with the text type. This includes Headings, Paragraphs,
// CodeBlock, CodeBlockTitle, ListItem and so on.
if cb.Type == BlockNodeText {
text.WriteString(cb.Text)
continue
}
content := cb.Search(scope)
maps.Insert(res, maps.All(content))
}
// a metric block and a file block have no text children; the title
// and the file name are attributes.
switch b.Type { //nolint:exhaustive // the other types index their text children
case BlockNodeMetricBlock:
if title, ok := b.Attrs[AttrTitle].(string); ok {
text.WriteString(title)
}
case BlockNodeFileBlock:
if name, ok := b.Attrs[AttrName].(string); ok {
text.WriteString(name)
}
}
if text.Len() != 0 {
if id, ok := b.UID(); ok && id != "" {
res[id] = scope.Block(id, string(b.Type), text.String())
}
}
return res
}
// Mark represents a mark, such as bold or italic.
type Mark struct {
// Type is the type of the mark, such as "bold", "italic", etc.

94
server/core/internal/document/node_test.go
View File

@ -4,7 +4,6 @@ import (
"encoding/json"
"testing"
"github.com/oxynote/oxynote/server/core/internal/search"
"github.com/rs/xid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -247,99 +246,6 @@ func Test_Block_Flatten(t *testing.T) {
}
}
func Test_RootBlock_Search(t *testing.T) {
t.Parallel()
documentID := xid.New()
rb := RootBlock{
Type: BlockNodeDoc,
Content: []Block{
{
Type: BlockNodeParagraph,
Attrs: Attributes{"uid": "p1"},
Content: []Block{
{Type: BlockNodeText, Text: "first"},
},
},
{
Type: BlockNodeBulletList,
Attrs: Attributes{"uid": "l1"},
Content: []Block{
{
Type: BlockNodeListItem,
Attrs: Attributes{"uid": "li1"},
Content: []Block{
{Type: BlockNodeText, Text: "nested"},
},
},
},
},
// marks split the text into multiple fragments; all of
// them must be indexed, not just the last one.
{
Type: BlockNodeParagraph,
Attrs: Attributes{"uid": "p2"},
Content: []Block{
{Type: BlockNodeText, Text: "plain "},
{Type: BlockNodeText, Text: "bold", Marks: []Mark{{Type: "bold"}}},
{Type: BlockNodeText, Text: " tail"},
},
},
// no uid: the text is not indexable.
{
Type: BlockNodeParagraph,
Content: []Block{
{Type: BlockNodeText, Text: "orphan"},
},
},
// no text: nothing to index.
{Type: BlockNodeHorizontalRule, Attrs: Attributes{"uid": "hr1"}},
// a metric block's title is an attribute, not a text child.
{
Type: BlockNodeMetricBlock,
Attrs: Attributes{"uid": "m1", "title": "Pizza Fridays"},
},
// an untitled metric block has nothing to index.
{
Type: BlockNodeMetricBlock,
Attrs: Attributes{"uid": "m2", "title": ""},
},
{
Type: BlockNodeMetricBlock,
Attrs: Attributes{"uid": "m3"},
},
// a file block's name is an attribute, not a text child.
{
Type: BlockNodeFileBlock,
Attrs: Attributes{"uid": "f1", "name": "quarterly-report.pdf", "size": 2048},
},
// a file block still uploading has no name to index.
{
Type: BlockNodeFileBlock,
Attrs: Attributes{"uid": "f2"},
},
},
}
scope := search.Scope{
OrganizationID: "org-1",
DocumentID: documentID,
BranchID: xid.New(),
BranchName: "draft",
}
res := rb.Search(scope)
assert.Equal(t, map[string]search.Block{
"p1": scope.Block("p1", "paragraph", "first"),
"li1": scope.Block("li1", "listItem", "nested"),
"p2": scope.Block("p2", "paragraph", "plain bold tail"),
"m1": scope.Block("m1", "metricBlock", "Pizza Fridays"),
"f1": scope.Block("f1", "fileBlock", "quarterly-report.pdf"),
}, res)
}
func Test_RootBlock_StripCommentMarks(t *testing.T) {
t.Parallel()

399
server/core/internal/search/client.go
View File

@ -1,399 +0,0 @@
// Package search indexes and searches documents through Meilisearch.
package search
import (
"context"
_ "embed"
"encoding/json"
"errors"
"fmt"
"maps"
"net/http"
"slices"
"strconv"
"strings"
"time"
"github.com/meilisearch/meilisearch-go"
"github.com/oxynote/oxynote/server/core/pkg/errutil"
)
//go:embed static/synonyms.json
var synonymsFile []byte
// ErrNotConfigured is returned when search is not configured on this
// deployment.
var ErrNotConfigured = errutil.New(http.StatusConflict, "search.not_configured", "search is not configured")
// _documentsIndex is the name of the documents index.
const _documentsIndex = "documents"
// _textAttribute is the indexed block attribute searched, cropped, and
// highlighted.
const _textAttribute = "text"
const (
// _searchCropLength is the number of words kept around a match in
// cropped search results.
_searchCropLength = 5
// _searchResultLimit caps the number of search hits returned.
_searchResultLimit = 20
// _taskPollInterval is how often an asynchronous Meilisearch task is
// polled while waiting for it to settle.
_taskPollInterval = 50 * time.Millisecond
// _setupTimeout bounds the index setup at boot.
_setupTimeout = time.Minute
// _writeTimeout bounds one batch of index writes, task waits included.
_writeTimeout = 2 * time.Minute
// _searchTimeout bounds one search request.
_searchTimeout = 10 * time.Second
)
// _displayedAttributes are the entry fields search hits carry.
var _displayedAttributes = []string{
"id",
"organizationId",
"documentId",
"branchId",
"branchName",
"branchDefault",
"type",
_textAttribute,
}
// _filterableAttributes are the entry fields searches and removals filter
// by.
var _filterableAttributes = []string{
"organizationId",
"documentId",
"branchId",
}
// Client is a Meilisearch client wrapper.
type Client struct {
meiliMan meilisearch.ServiceManager
}
// NewClient creates a new Meilisearch client wrapper. A nil meiliMan means
// search is not configured on this deployment: the client is returned
// without touching Meilisearch and every call on it reports
// ErrNotConfigured.
func NewClient(
ctx context.Context,
meiliMan meilisearch.ServiceManager,
) (*Client, error) {
c := &Client{
meiliMan: meiliMan,
}
if !c.Configured() {
return c, nil
}
if err := c.ensureIndex(ctx); err != nil {
return nil, fmt.Errorf("ensuring documents index: %w", err)
}
return c, nil
}
// Configured reports whether search is configured on this deployment.
func (c *Client) Configured() bool {
return c.meiliMan != nil
}
// ensureIndex ensures the documents index exists and runs with the
// declared settings.
func (c *Client) ensureIndex(ctx context.Context) error {
ctx, cancel := context.WithTimeout(ctx, _setupTimeout)
defer cancel()
_, err := c.meiliMan.GetIndexWithContext(ctx, _documentsIndex)
switch merr, ok := errors.AsType[*meilisearch.Error](err); {
case err == nil:
// the index is already there.
case ok && merr.MeilisearchApiError.Code == "index_not_found":
task, cerr := c.meiliMan.CreateIndexWithContext(ctx, &meilisearch.IndexConfig{
Uid: _documentsIndex,
PrimaryKey: "id",
})
if cerr != nil {
return fmt.Errorf("creating documents index: %w", cerr)
}
if cerr = c.awaitTask(ctx, task); cerr != nil {
return fmt.Errorf("creating documents index: %w", cerr)
}
default:
return fmt.Errorf("getting documents index: %w", err)
}
// the settings are checked on every boot rather than only when the
// index is created: an index whose settings task failed would otherwise
// stay broken for good, and new synonyms would never reach an existing
// deployment. They are sent only when they differ, since a searchable
// or filterable attribute change re-indexes every entry.
want, err := indexSettings()
if err != nil {
return err
}
have, err := c.meiliMan.Index(_documentsIndex).GetSettingsWithContext(ctx)
if err != nil {
return fmt.Errorf("getting settings: %w", err)
}
if settingsMatch(have, want) {
return nil
}
task, err := c.meiliMan.Index(_documentsIndex).UpdateSettingsWithContext(ctx, want)
if err != nil {
return fmt.Errorf("updating settings: %w", err)
}
if err = c.awaitTask(ctx, task); err != nil {
return fmt.Errorf("updating settings: %w", err)
}
return nil
}
// indexSettings returns the settings the documents index runs with. Only
// the fields it sets are sent; everything else keeps Meilisearch's default.
func indexSettings() (*meilisearch.Settings, error) {
var synonyms map[string][]string
if err := json.Unmarshal(synonymsFile, &synonyms); err != nil {
return nil, fmt.Errorf("unmarshaling synonyms: %w", err)
}
return &meilisearch.Settings{
SearchableAttributes: []string{_textAttribute},
DisplayedAttributes: _displayedAttributes,
FilterableAttributes: _filterableAttributes,
Synonyms: synonyms,
}, nil
}
// settingsMatch reports whether the index already runs with every field
// want sets. Attribute lists are compared as sets, since Meilisearch does
// not promise their order back.
func settingsMatch(have, want *meilisearch.Settings) bool {
sameSet := func(a, b []string) bool {
a, b = slices.Sorted(slices.Values(a)), slices.Sorted(slices.Values(b))
return slices.Equal(a, b)
}
return sameSet(have.SearchableAttributes, want.SearchableAttributes) &&
sameSet(have.DisplayedAttributes, want.DisplayedAttributes) &&
sameSet(have.FilterableAttributes, want.FilterableAttributes) &&
maps.EqualFunc(have.Synonyms, want.Synonyms, sameSet)
}
// awaitTask blocks until an asynchronous Meilisearch task settles and reports
// a rejected task as an error, so a failed index or settings update surfaces
// at boot instead of leaving the index quietly misconfigured.
func (c *Client) awaitTask(ctx context.Context, info *meilisearch.TaskInfo) error {
task, err := c.meiliMan.WaitForTaskWithContext(ctx, info.TaskUID, _taskPollInterval)
if err != nil {
return fmt.Errorf("waiting for task %d: %w", info.TaskUID, err)
}
if task.Status != meilisearch.TaskStatusSucceeded {
return fmt.Errorf(
"task %d finished with status %q: %s (%s)",
info.TaskUID,
task.Status,
task.Error.Message,
task.Error.Code,
)
}
return nil
}
// settle reports the outcome of one index write: the request error when the
// call itself failed, otherwise the task's result once Meilisearch has
// applied it.
func (c *Client) settle(ctx context.Context, what string, task *meilisearch.TaskInfo, err error) error {
if err != nil {
return fmt.Errorf("%s: %w", what, err)
}
if err := c.awaitTask(ctx, task); err != nil {
return fmt.Errorf("%s: %w", what, err)
}
return nil
}
// SearchDocuments searches documents in the index.
func (c *Client) SearchDocuments(ctx context.Context, organizationID, query string) ([]byte, error) {
if !c.Configured() {
return nil, ErrNotConfigured
}
ctx, cancel := context.WithTimeout(ctx, _searchTimeout)
defer cancel()
res, err := c.meiliMan.Index(_documentsIndex).SearchWithContext(ctx, query, &meilisearch.SearchRequest{
AttributesToSearchOn: []string{_textAttribute},
AttributesToCrop: []string{_textAttribute},
AttributesToHighlight: []string{_textAttribute},
Filter: fmt.Sprintf("organizationId = %q", organizationID),
HighlightPreTag: "<mark>",
HighlightPostTag: "</mark>",
CropLength: _searchCropLength,
Limit: _searchResultLimit,
})
if err != nil {
return nil, fmt.Errorf("searching documents: %w", err)
}
hits := make([]json.RawMessage, 0, len(res.Hits))
for _, hit := range res.Hits {
hits = append(
hits,
hit["_formatted"],
)
}
data, err := json.Marshal(hits)
if err != nil {
return nil, fmt.Errorf("marshaling search results: %w", err)
}
return data, nil
}
// SearchDocumentBlocks returns the raw matching blocks for the
// query, scoped to the organization. Unlike SearchDocuments it
// returns plain block values (no highlight markup or cropping) so
// callers — the AI assistant's search tool — can consume the text
// directly.
func (c *Client) SearchDocumentBlocks(ctx context.Context, organizationID, query string, limit int) ([]Block, error) {
if !c.Configured() {
return nil, ErrNotConfigured
}
ctx, cancel := context.WithTimeout(ctx, _searchTimeout)
defer cancel()
res, err := c.meiliMan.Index(_documentsIndex).SearchWithContext(ctx, query, &meilisearch.SearchRequest{
AttributesToSearchOn: []string{_textAttribute},
Filter: fmt.Sprintf("organizationId = %q", organizationID),
Limit: int64(limit),
})
if err != nil {
return nil, fmt.Errorf("searching document blocks: %w", err)
}
blocks := make([]Block, 0, len(res.Hits))
for _, hit := range res.Hits {
var b Block
if err := hit.DecodeInto(&b); err != nil {
return nil, fmt.Errorf("decoding search hit: %w", err)
}
blocks = append(blocks, b)
}
return blocks, nil
}
// ReplaceDocumentBlocks replaces document blocks in the index based on the
// provided differences. Every write is awaited, so a batch Meilisearch
// rejects is reported rather than silently dropped.
func (c *Client) ReplaceDocumentBlocks(ctx context.Context, bd BlocksDifference) error {
if !c.Configured() {
return ErrNotConfigured
}
ctx, cancel := context.WithTimeout(ctx, _writeTimeout)
defer cancel()
idx := c.meiliMan.Index(_documentsIndex)
// every entry is complete, so added and updated ones alike are sent as
// one add-or-replace request.
if docs := slices.Concat(bd.Added, bd.Updated); len(docs) != 0 {
task, err := idx.AddDocumentsWithContext(ctx, docs, nil)
if err = c.settle(ctx, "adding documents", task, err); err != nil {
return err
}
}
if len(bd.Removed) != 0 {
ids := make([]string, 0, len(bd.Removed))
for _, block := range bd.Removed {
ids = append(ids, block.ID)
}
task, err := idx.DeleteDocumentsWithContext(ctx, ids, nil)
if err = c.settle(ctx, "deleting documents", task, err); err != nil {
return err
}
}
if len(bd.RemovedBranches) != 0 {
ids := make([]string, 0, len(bd.RemovedBranches))
for _, br := range bd.RemovedBranches {
ids = append(ids, br.BranchID.String())
}
if err := c.deleteByFilter(ctx, idx, "branchId", ids); err != nil {
return fmt.Errorf("deleting branches by filter: %w", err)
}
}
if len(bd.RemovedDocuments) != 0 {
ids := make([]string, 0, len(bd.RemovedDocuments))
for _, id := range bd.RemovedDocuments {
ids = append(ids, id.String())
}
if err := c.deleteByFilter(ctx, idx, "documentId", ids); err != nil {
return fmt.Errorf("deleting documents by filter: %w", err)
}
}
if len(bd.RemovedOrganizations) != 0 {
if err := c.deleteByFilter(ctx, idx, "organizationId", bd.RemovedOrganizations); err != nil {
return fmt.Errorf("deleting organizations by filter: %w", err)
}
}
return nil
}
// deleteByFilter removes every entry whose field is one of the ids and
// waits for the removal to apply.
func (c *Client) deleteByFilter(ctx context.Context, idx meilisearch.IndexManager, field string, ids []string) error {
quoted := make([]string, 0, len(ids))
for _, id := range ids {
quoted = append(quoted, strconv.Quote(id))
}
filter := fmt.Sprintf("%s IN [%s]", field, strings.Join(quoted, ", "))
task, err := idx.DeleteDocumentsByFilterWithContext(ctx, filter, nil)
return c.settle(ctx, "deleting by "+field, task, err)
}

606
server/core/internal/search/client_test.go
View File

@ -1,606 +0,0 @@
package search
import (
"context"
"encoding/json"
"fmt"
"slices"
"testing"
"time"
"github.com/meilisearch/meilisearch-go"
mock "github.com/oxynote/oxynote/server/core/internal/_mock"
"github.com/rs/xid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// indexNotFoundErr builds the meilisearch error ensureIndex treats as a
// missing index.
func indexNotFoundErr() *meilisearch.Error {
merr := &meilisearch.Error{}
merr.MeilisearchApiError.Code = "index_not_found"
return merr
}
// stubManagers wires a service-manager mock whose Index method returns the
// given index-manager mock.
func stubManagers(idx *mock.MeiliIndexManager) *mock.MeiliServiceManager {
return &mock.MeiliServiceManager{
IndexFunc: func(string) meilisearch.IndexManager {
return idx
},
}
}
func Test_NewClient(t *testing.T) {
t.Parallel()
// a nil service manager is the disabled mode: nothing is set up and
// every call refuses with the sentinel.
t.Run("Unconfigured client", func(t *testing.T) {
t.Parallel()
c, err := NewClient(context.Background(), nil)
require.NoError(t, err)
require.NotNil(t, c)
assert.False(t, c.Configured())
data, serr := c.SearchDocuments(context.Background(), "org1", "query")
assert.ErrorIs(t, serr, ErrNotConfigured)
assert.Nil(t, data)
blocks, serr := c.SearchDocumentBlocks(context.Background(), "org1", "query", 5)
assert.ErrorIs(t, serr, ErrNotConfigured)
assert.Nil(t, blocks)
assert.ErrorIs(t, c.ReplaceDocumentBlocks(context.Background(), BlocksDifference{}), ErrNotConfigured)
})
type check func(*testing.T, *mock.MeiliServiceManager, *mock.MeiliIndexManager, error)
checks := func(cc ...check) []check { return cc }
hasError := func(expect bool) check {
return func(t *testing.T, _ *mock.MeiliServiceManager, _ *mock.MeiliIndexManager, err error) {
if expect {
require.Error(t, err)
return
}
require.NoError(t, err)
}
}
wasCreateIndexCalled := func(count int) check {
return func(t *testing.T, svc *mock.MeiliServiceManager, _ *mock.MeiliIndexManager, _ error) {
ff := svc.CreateIndexWithContextCalls()
require.Len(t, ff, count)
if count == 0 {
return
}
assert.Equal(t, "documents", ff[0].Config.Uid)
assert.Equal(t, "id", ff[0].Config.PrimaryKey)
}
}
wasSettingsUpdated := func(count int) check {
return func(t *testing.T, _ *mock.MeiliServiceManager, idx *mock.MeiliIndexManager, _ error) {
ff := idx.UpdateSettingsWithContextCalls()
require.Len(t, ff, count)
if count == 0 {
return
}
assert.Equal(t, []string{"text"}, ff[0].Request.SearchableAttributes)
assert.Equal(t, _displayedAttributes, ff[0].Request.DisplayedAttributes)
assert.Equal(t, _filterableAttributes, ff[0].Request.FilterableAttributes)
assert.NotEmpty(t, ff[0].Request.Synonyms)
}
}
// current is what a settled index reports: the declared settings, with
// the attribute lists in an order of Meilisearch's choosing.
current := func() *meilisearch.Settings {
want, err := indexSettings()
require.NoError(t, err)
have := *want
have.DisplayedAttributes = slices.Clone(want.DisplayedAttributes)
slices.Reverse(have.DisplayedAttributes)
return &have
}
// fresh is what a just-created index reports.
fresh := func() *meilisearch.Settings {
return &meilisearch.Settings{
SearchableAttributes: []string{"*"},
DisplayedAttributes: []string{"*"},
}
}
tests := map[string]struct {
GetIndexErr error
CreateErr error
Settings *meilisearch.Settings
GetSettingsErr error
SettingsErr error
WaitErr error
TaskStatus meilisearch.TaskStatus
Checks []check
}{
"Existing index with the declared settings is left alone": {
Settings: current(),
Checks: checks(
hasError(false),
wasCreateIndexCalled(0),
wasSettingsUpdated(0),
),
},
"Existing index with stale settings gets them updated": {
Settings: func() *meilisearch.Settings {
st := current()
st.FilterableAttributes = []string{"organizationId", "documentId"}
return st
}(),
Checks: checks(
hasError(false),
wasCreateIndexCalled(0),
wasSettingsUpdated(1),
),
},
"Existing index with stale synonyms gets them updated": {
Settings: func() *meilisearch.Settings {
st := current()
st.Synonyms = map[string][]string{"2fa": {"mfa"}}
return st
}(),
Checks: checks(
hasError(false),
wasSettingsUpdated(1),
),
},
"Settings fetch failure fails": {
GetSettingsErr: assert.AnError,
Checks: checks(
hasError(true),
wasSettingsUpdated(0),
),
},
"Rejected settings task fails": {
Settings: fresh(),
TaskStatus: meilisearch.TaskStatusFailed,
Checks: checks(
hasError(true),
wasCreateIndexCalled(0),
wasSettingsUpdated(1),
),
},
"Task wait failure fails": {
Settings: fresh(),
WaitErr: assert.AnError,
Checks: checks(
hasError(true),
),
},
"Missing index is created with settings": {
GetIndexErr: indexNotFoundErr(),
Settings: fresh(),
Checks: checks(
hasError(false),
wasCreateIndexCalled(1),
wasSettingsUpdated(1),
),
},
"Other meilisearch errors fail": {
GetIndexErr: &meilisearch.Error{},
Checks: checks(
hasError(true),
wasCreateIndexCalled(0),
),
},
"Non-meilisearch errors fail": {
GetIndexErr: assert.AnError,
Checks: checks(
hasError(true),
wasCreateIndexCalled(0),
),
},
"Index creation failure fails": {
GetIndexErr: indexNotFoundErr(),
CreateErr: assert.AnError,
Checks: checks(
hasError(true),
wasSettingsUpdated(0),
),
},
"Settings update failure fails": {
GetIndexErr: indexNotFoundErr(),
Settings: fresh(),
SettingsErr: assert.AnError,
Checks: checks(
hasError(true),
wasSettingsUpdated(1),
),
},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
t.Parallel()
idx := &mock.MeiliIndexManager{
GetSettingsWithContextFunc: func(context.Context) (*meilisearch.Settings, error) {
return tc.Settings, tc.GetSettingsErr
},
UpdateSettingsWithContextFunc: func(context.Context, *meilisearch.Settings) (*meilisearch.TaskInfo, error) {
return &meilisearch.TaskInfo{TaskUID: 2}, tc.SettingsErr
},
}
svc := stubManagers(idx)
svc.GetIndexWithContextFunc = func(context.Context, string) (*meilisearch.IndexResult, error) {
return nil, tc.GetIndexErr
}
svc.CreateIndexWithContextFunc = func(context.Context, *meilisearch.IndexConfig) (*meilisearch.TaskInfo, error) {
return &meilisearch.TaskInfo{TaskUID: 1}, tc.CreateErr
}
svc.WaitForTaskWithContextFunc = func(context.Context, int64, time.Duration) (*meilisearch.Task, error) {
status := meilisearch.TaskStatusSucceeded
if tc.TaskStatus != "" {
status = tc.TaskStatus
}
return &meilisearch.Task{Status: status}, tc.WaitErr
}
client, err := NewClient(context.Background(), svc)
for _, ch := range tc.Checks {
ch(t, svc, idx, err)
}
if err == nil {
assert.NotNil(t, client)
}
})
}
}
// newTestClient builds a Client wired straight to the given index mock,
// bypassing ensureIndex. Every task the index reports settles with the
// given status.
func newTestClient(idx *mock.MeiliIndexManager, status meilisearch.TaskStatus) *Client {
svc := stubManagers(idx)
svc.WaitForTaskWithContextFunc = func(context.Context, int64, time.Duration) (*meilisearch.Task, error) {
task := &meilisearch.Task{Status: status}
task.Error.Message = "boom"
task.Error.Code = "invalid_document_id"
return task, nil
}
return &Client{meiliMan: svc}
}
func Test_Client_SearchDocuments(t *testing.T) {
t.Parallel()
t.Run("Formatted hits are returned as JSON", func(t *testing.T) {
t.Parallel()
var captured *meilisearch.SearchRequest
idx := &mock.MeiliIndexManager{
SearchWithContextFunc: func(_ context.Context, _ string, request *meilisearch.SearchRequest) (*meilisearch.SearchResponse, error) {
captured = request
return &meilisearch.SearchResponse{
Hits: meilisearch.Hits{
{"_formatted": json.RawMessage(`{"id": "b1", "text": "<mark>hello</mark>"}`)},
},
}, nil
},
}
data, err := newTestClient(idx, meilisearch.TaskStatusSucceeded).SearchDocuments(context.Background(), "org-1", "hello")
require.NoError(t, err)
assert.JSONEq(t, `[{"id": "b1", "text": "<mark>hello</mark>"}]`, string(data))
require.NotNil(t, captured)
assert.Equal(t, `organizationId = "org-1"`, captured.Filter)
assert.Equal(t, []string{"text"}, captured.AttributesToSearchOn)
assert.Equal(t, int64(5), captured.CropLength)
assert.Equal(t, int64(20), captured.Limit)
assert.Equal(t, "<mark>", captured.HighlightPreTag)
})
t.Run("Search failure is propagated", func(t *testing.T) {
t.Parallel()
idx := &mock.MeiliIndexManager{
SearchWithContextFunc: func(context.Context, string, *meilisearch.SearchRequest) (*meilisearch.SearchResponse, error) {
return nil, assert.AnError
},
}
_, err := newTestClient(idx, meilisearch.TaskStatusSucceeded).SearchDocuments(context.Background(), "org-1", "hello")
require.Error(t, err)
})
}
func Test_Client_SearchDocumentBlocks(t *testing.T) {
t.Parallel()
documentID := xid.New()
t.Run("Hits are decoded into blocks", func(t *testing.T) {
t.Parallel()
var captured *meilisearch.SearchRequest
idx := &mock.MeiliIndexManager{
SearchWithContextFunc: func(_ context.Context, _ string, request *meilisearch.SearchRequest) (*meilisearch.SearchResponse, error) {
captured = request
return &meilisearch.SearchResponse{
Hits: meilisearch.Hits{
{
"id": json.RawMessage(`"b1"`),
"organizationId": json.RawMessage(`"org-1"`),
"documentId": json.RawMessage(`"` + documentID.String() + `"`),
"type": json.RawMessage(`"paragraph"`),
"text": json.RawMessage(`"hello"`),
},
},
}, nil
},
}
blocks, err := newTestClient(idx, meilisearch.TaskStatusSucceeded).SearchDocumentBlocks(context.Background(), "org-1", "hello", 7)
require.NoError(t, err)
assert.Equal(t, []Block{{
ID: "b1",
OrganizationID: "org-1",
DocumentID: documentID,
Type: "paragraph",
Text: "hello",
}}, blocks)
require.NotNil(t, captured)
assert.Equal(t, `organizationId = "org-1"`, captured.Filter)
assert.Equal(t, int64(7), captured.Limit)
})
t.Run("Undecodable hit fails", func(t *testing.T) {
t.Parallel()
idx := &mock.MeiliIndexManager{
SearchWithContextFunc: func(context.Context, string, *meilisearch.SearchRequest) (*meilisearch.SearchResponse, error) {
return &meilisearch.SearchResponse{
Hits: meilisearch.Hits{
{"id": json.RawMessage(`42`)},
},
}, nil
},
}
_, err := newTestClient(idx, meilisearch.TaskStatusSucceeded).SearchDocumentBlocks(context.Background(), "org-1", "hello", 7)
require.Error(t, err)
})
t.Run("Search failure is propagated", func(t *testing.T) {
t.Parallel()
idx := &mock.MeiliIndexManager{
SearchWithContextFunc: func(context.Context, string, *meilisearch.SearchRequest) (*meilisearch.SearchResponse, error) {
return nil, assert.AnError
},
}
_, err := newTestClient(idx, meilisearch.TaskStatusSucceeded).SearchDocumentBlocks(context.Background(), "org-1", "hello", 7)
require.Error(t, err)
})
}
func Test_Client_ReplaceDocumentBlocks(t *testing.T) {
t.Parallel()
type check func(*testing.T, *mock.MeiliIndexManager, error)
checks := func(cc ...check) []check { return cc }
hasError := func(expect bool) check {
return func(t *testing.T, _ *mock.MeiliIndexManager, err error) {
if expect {
require.Error(t, err)
return
}
require.NoError(t, err)
}
}
wasAddCalled := func(count int) check {
return func(t *testing.T, idx *mock.MeiliIndexManager, _ error) {
require.Len(t, idx.AddDocumentsWithContextCalls(), count)
}
}
wasDeleteCalled := func(count int) check {
return func(t *testing.T, idx *mock.MeiliIndexManager, _ error) {
require.Len(t, idx.DeleteDocumentsWithContextCalls(), count)
}
}
wasDeleteByFilterCalled := func(count int) check {
return func(t *testing.T, idx *mock.MeiliIndexManager, _ error) {
require.Len(t, idx.DeleteDocumentsByFilterWithContextCalls(), count)
}
}
hasFilter := func(filter string) check {
return func(t *testing.T, idx *mock.MeiliIndexManager, _ error) {
ff := idx.DeleteDocumentsByFilterWithContextCalls()
require.NotEmpty(t, ff)
assert.Equal(t, filter, ff[0].Filter)
}
}
docID1, docID2 := xid.New(), xid.New()
branchID1, branchID2 := xid.New(), xid.New()
fullDiff := BlocksDifference{
Added: []Block{{ID: "a"}},
Updated: []Block{{ID: "u"}},
Removed: []Block{{ID: "r1"}, {ID: "r2"}},
}
tests := map[string]struct {
Diff BlocksDifference
TaskStatus meilisearch.TaskStatus
AddErr error
DeleteErr error
DeleteFilterErr error
Checks []check
}{
"Empty difference makes no calls": {
Checks: checks(
hasError(false),
wasAddCalled(0),
wasDeleteCalled(0),
wasDeleteByFilterCalled(0),
),
},
"Removed branches are cleared by filter": {
Diff: BlocksDifference{RemovedBranches: []BranchRemoval{
{DocumentID: docID1, BranchID: branchID1},
{DocumentID: docID1, BranchID: branchID2},
}},
Checks: checks(
hasError(false),
wasDeleteCalled(0),
wasDeleteByFilterCalled(1),
hasFilter(fmt.Sprintf("branchId IN [%q, %q]", branchID1, branchID2)),
),
},
"Removed documents are cleared by filter": {
Diff: BlocksDifference{RemovedDocuments: []xid.ID{docID1, docID2}},
Checks: checks(
hasError(false),
wasDeleteCalled(0),
wasDeleteByFilterCalled(1),
hasFilter(fmt.Sprintf("documentId IN [%q, %q]", docID1, docID2)),
),
},
"Removed organizations are cleared by filter": {
Diff: BlocksDifference{RemovedOrganizations: []string{"org-1", "org-2"}},
Checks: checks(
hasError(false),
wasDeleteCalled(0),
wasDeleteByFilterCalled(1),
hasFilter(`organizationId IN ["org-1", "org-2"]`),
),
},
"Delete by filter failure is propagated": {
Diff: BlocksDifference{RemovedDocuments: []xid.ID{docID1}},
DeleteFilterErr: assert.AnError,
Checks: checks(
hasError(true),
wasDeleteByFilterCalled(1),
),
},
// added and updated entries are complete either way, so one
// add-or-replace request carries both.
"Full difference sends one add and one delete": {
Diff: fullDiff,
Checks: checks(
hasError(false),
wasAddCalled(1),
wasDeleteCalled(1),
func(t *testing.T, idx *mock.MeiliIndexManager, _ error) {
assert.Equal(t, []Block{{ID: "a"}, {ID: "u"}}, idx.AddDocumentsWithContextCalls()[0].DocumentsPtr)
assert.Equal(t, []string{"r1", "r2"}, idx.DeleteDocumentsWithContextCalls()[0].Identifiers)
},
),
},
"Add failure short-circuits": {
Diff: fullDiff,
AddErr: assert.AnError,
Checks: checks(
hasError(true),
wasDeleteCalled(0),
),
},
"Delete failure is propagated": {
Diff: fullDiff,
DeleteErr: assert.AnError,
Checks: checks(
hasError(true),
wasAddCalled(1),
),
},
// the request is accepted but the task fails later; the batch is
// reported so the job is retried rather than dropped.
"Failed task is reported with its cause": {
Diff: fullDiff,
TaskStatus: meilisearch.TaskStatusFailed,
Checks: checks(
hasError(true),
wasAddCalled(1),
wasDeleteCalled(0),
func(t *testing.T, _ *mock.MeiliIndexManager, err error) {
assert.ErrorContains(t, err, "boom (invalid_document_id)")
},
),
},
"Calls run under a deadline": {
Diff: fullDiff,
Checks: checks(
hasError(false),
func(t *testing.T, idx *mock.MeiliIndexManager, _ error) {
_, ok := idx.AddDocumentsWithContextCalls()[0].Ctx.Deadline()
assert.True(t, ok)
},
),
},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
t.Parallel()
idx := &mock.MeiliIndexManager{
AddDocumentsWithContextFunc: func(context.Context, any, *meilisearch.DocumentOptions) (*meilisearch.TaskInfo, error) {
return &meilisearch.TaskInfo{TaskUID: 1}, tc.AddErr
},
DeleteDocumentsWithContextFunc: func(context.Context, []string, *meilisearch.DocumentOptions) (*meilisearch.TaskInfo, error) {
return &meilisearch.TaskInfo{TaskUID: 2}, tc.DeleteErr
},
DeleteDocumentsByFilterWithContextFunc: func(context.Context, any, *meilisearch.DocumentOptions) (*meilisearch.TaskInfo, error) {
return &meilisearch.TaskInfo{TaskUID: 3}, tc.DeleteFilterErr
},
}
status := meilisearch.TaskStatusSucceeded
if tc.TaskStatus != "" {
status = tc.TaskStatus
}
err := newTestClient(idx, status).ReplaceDocumentBlocks(context.Background(), tc.Diff)
for _, ch := range tc.Checks {
ch(t, idx, err)
}
})
}
}

173
server/core/internal/search/document.go
View File

@ -1,13 +1,16 @@
package search
import (
"database/sql/driver"
"encoding/json"
"fmt"
"strings"
"github.com/oxynote/oxynote/server/core/internal/document"
"github.com/rs/xid"
)
// _documentNameUID is the uid of the synthetic entry carrying a branch's
// document name. Content blocks never use it.
const _documentNameUID = "docname"
// Block represents a information block. It can be a paragraph, heading, or
// any other type of content in a document.
type Block struct {
@ -37,6 +40,45 @@ type Block struct {
Text string `json:"text"`
}
// record is the shape a block is written to the index in. bleve walks
// struct fields by reflection and would take an xid's byte array for a
// list of numbers, so the ids are flattened to strings first.
type record struct {
// OrganizationID specifies the organization the block belongs to.
OrganizationID string `json:"organizationId"`
// DocumentID specifies the document the block belongs to.
DocumentID string `json:"documentId"`
// BranchID specifies the branch the block belongs to.
BranchID string `json:"branchId"`
// BranchName specifies the name of the branch, kept for display only.
BranchName string `json:"branchName"`
// BranchDefault indicates whether the branch is the document's default.
BranchDefault bool `json:"branchDefault"`
// Type specifies the block type (e.g., "paragraph", "heading").
Type string `json:"type"`
// Text specifies the searchable text of the block.
Text string `json:"text"`
}
// record flattens the block for indexing.
func (b Block) record() record {
return record{
OrganizationID: b.OrganizationID,
DocumentID: b.DocumentID.String(),
BranchID: b.BranchID.String(),
BranchName: b.BranchName,
BranchDefault: b.BranchDefault,
Type: b.Type,
Text: b.Text,
}
}
// Scope is the branch every block of one indexing pass belongs to.
type Scope struct {
// OrganizationID is the organization owning the document.
@ -71,93 +113,68 @@ func (s Scope) Block(uid, typ, text string) Block {
}
}
// BranchRemoval names a branch whose every block is removed. The document
// id is carried so the removal is ordered with the document's other jobs.
type BranchRemoval struct {
// DocumentID is the document the branch belonged to.
DocumentID xid.ID `json:"documentId"`
// entries collects the index entries of the block and its descendants
// into res, keyed by block uid.
func (s Scope) entries(b document.Block, res map[string]Block) {
var text strings.Builder
// BranchID is the removed branch.
BranchID xid.ID `json:"branchId"`
}
// BlocksDifference represents the differences between two slices of Blocks.
type BlocksDifference struct {
// Updated contains blocks that have been modified.
Updated []Block `json:"updated"`
// Added contains blocks that have been added.
Added []Block `json:"added"`
// Removed contains blocks that have been removed.
Removed []Block `json:"removed"`
// RemovedBranches contains branches whose every block is removed.
// Deleting a branch drops its content row, so the index is cleared by
// branch instead of by block.
RemovedBranches []BranchRemoval `json:"removedBranches"`
// RemovedDocuments contains documents whose every block is removed.
// Deleting a document cascades to its descendants in the database,
// which makes enumerating their blocks impossible after the fact, so
// the index is cleared by document instead.
RemovedDocuments []xid.ID `json:"removedDocuments"`
// RemovedOrganizations contains organizations whose every block is
// removed. Organizations are deleted outside of core, so this is the
// only handle left on the content that went with them.
RemovedOrganizations []string `json:"removedOrganizations"`
}
// BlocksDiff compares two slices of Blocks and returns the differences
// as a BlocksDifference struct.
func BlocksDiff(a, b map[string]Block) BlocksDifference {
var diff BlocksDifference
// Check for added and updated blocks.
for id, blockB := range b {
blockA, exists := a[id]
if !exists {
diff.Added = append(diff.Added, blockB)
} else if blockA != blockB {
diff.Updated = append(diff.Updated, blockB)
for _, cb := range b.Content {
// every parent node holding text has a child of the text type,
// headings, paragraphs, code blocks and list items included.
if cb.Type == document.BlockNodeText {
text.WriteString(cb.Text)
continue
}
s.entries(cb, res)
}
// Check for removed blocks.
for id, blockA := range a {
if _, exists := b[id]; !exists {
diff.Removed = append(diff.Removed, blockA)
// a metric, file or image block has no text children; what describes
// it sits in its attributes.
switch b.Type { //nolint:exhaustive // the other types index their text children
case document.BlockNodeMetricBlock:
if title, ok := b.Attrs[document.AttrTitle].(string); ok {
text.WriteString(title)
}
case document.BlockNodeFileBlock:
if name, ok := b.Attrs[document.AttrName].(string); ok {
text.WriteString(name)
}
case document.BlockNodeImageBlock:
alt, _ := b.Attrs[document.AttrAlt].(string)
title, _ := b.Attrs[document.AttrTitle].(string)
text.WriteString(strings.TrimSpace(alt + " " + title))
}
return diff
}
// Scan implements the sql.Scanner interface for BlocksDifference.
func (bd *BlocksDifference) Scan(value any) error {
if value == nil {
return nil
if text.Len() == 0 {
return
}
bytes, ok := value.([]byte)
if !ok {
return fmt.Errorf("failed to scan BlocksDifference: expected []byte, got %T", value)
if id, ok := b.UID(); ok && id != "" {
res[id] = s.Block(id, string(b.Type), text.String())
}
}
// Entries converts a document branch into its index entries, keyed by
// block uid. A synthetic entry carries the document name; its key is the
// document id, which no content block shares.
func Entries(doc document.Document) map[string]Block {
scope := Scope{
OrganizationID: doc.OrganizationID,
DocumentID: doc.ID,
BranchID: doc.BranchID,
BranchName: doc.BranchName,
BranchDefault: doc.Default,
}
return json.Unmarshal(bytes, bd)
}
res := make(map[string]Block)
// Value implements the driver.Valuer interface for BlocksDifference.
func (bd BlocksDifference) Value() (driver.Value, error) {
return json.Marshal(bd)
}
for _, b := range doc.Content.Content {
scope.entries(b, res)
}
// DocumentSearchJob represents a search index update job.
type DocumentSearchJob struct {
// ID is the unique identifier for the search job.
ID int64 `db:"id"`
res[doc.ID.String()] = scope.Block(_documentNameUID, "document", doc.DocumentName)
// BlockDiff contains the differences in blocks for the document.
BlockDiff BlocksDifference `db:"block_diff"`
return res
}

239
server/core/internal/search/document_test.go
View File

@ -1,96 +1,29 @@
package search
import (
"encoding/json"
"testing"
"github.com/oxynote/oxynote/server/core/internal/document"
"github.com/rs/xid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_BlocksDiff(t *testing.T) {
t.Parallel()
// empty inputs
empty := BlocksDiff(nil, nil)
assert.Empty(t, empty.Added)
assert.Empty(t, empty.Updated)
assert.Empty(t, empty.Removed)
// added, updated and removed blocks
kept := Block{ID: "kept", Text: "same"}
changed := Block{ID: "changed", Text: "old"}
changedNew := Block{ID: "changed", Text: "new"}
removed := Block{ID: "removed", Text: "gone"}
added := Block{ID: "added", Text: "fresh"}
diff := BlocksDiff(
map[string]Block{"kept": kept, "changed": changed, "removed": removed},
map[string]Block{"kept": kept, "changed": changedNew, "added": added},
)
assert.Equal(t, []Block{added}, diff.Added)
assert.Equal(t, []Block{changedNew}, diff.Updated)
assert.Equal(t, []Block{removed}, diff.Removed)
}
// stubBlocksDifference builds a difference carrying one block of each kind.
func stubBlocksDifference() BlocksDifference {
return BlocksDifference{
Added: []Block{{ID: "a", Text: "added"}},
Updated: []Block{{ID: "u", Text: "updated"}},
Removed: []Block{{ID: "r", Text: "removed"}},
}
}
func Test_BlocksDifference_Value(t *testing.T) {
t.Parallel()
orig := stubBlocksDifference()
val, err := orig.Value()
require.NoError(t, err)
data, ok := val.([]byte)
require.True(t, ok)
exp, err := json.Marshal(orig)
require.NoError(t, err)
assert.JSONEq(t, string(exp), string(data))
}
func Test_BlocksDifference_Scan(t *testing.T) {
t.Parallel()
orig := stubBlocksDifference()
val, err := orig.Value()
require.NoError(t, err)
var decoded BlocksDifference
require.NoError(t, decoded.Scan(val))
assert.Equal(t, orig, decoded)
require.NoError(t, decoded.Scan(nil))
assert.Equal(t, orig, decoded, "nil scan should leave the value untouched")
assert.Error(t, decoded.Scan("not bytes"))
assert.Error(t, decoded.Scan([]byte(`{not json`)))
}
func Test_Scope_Block(t *testing.T) {
t.Parallel()
scope := Scope{
// stubScope builds a scope on a draft branch.
func stubScope() Scope {
return Scope{
OrganizationID: "org-1",
DocumentID: xid.New(),
BranchID: xid.New(),
BranchName: "draft",
BranchDefault: false,
}
}
func Test_Scope_Block(t *testing.T) {
t.Parallel()
scope := stubScope()
assert.Equal(t, Block{
ID: scope.BranchID.String() + "-p1",
@ -103,3 +36,155 @@ func Test_Scope_Block(t *testing.T) {
Text: "hello",
}, scope.Block("p1", "paragraph", "hello"))
}
func Test_Scope_entries(t *testing.T) {
t.Parallel()
blocks := []document.Block{
{
Type: document.BlockNodeParagraph,
Attrs: document.Attributes{"uid": "p1"},
Content: []document.Block{
{Type: document.BlockNodeText, Text: "first"},
},
},
{
Type: document.BlockNodeBulletList,
Attrs: document.Attributes{"uid": "l1"},
Content: []document.Block{
{
Type: document.BlockNodeListItem,
Attrs: document.Attributes{"uid": "li1"},
Content: []document.Block{
{Type: document.BlockNodeText, Text: "nested"},
},
},
},
},
// marks split the text into multiple fragments; all of them are
// indexed, not just the last one.
{
Type: document.BlockNodeParagraph,
Attrs: document.Attributes{"uid": "p2"},
Content: []document.Block{
{Type: document.BlockNodeText, Text: "plain "},
{Type: document.BlockNodeText, Text: "bold", Marks: []document.Mark{{Type: "bold"}}},
{Type: document.BlockNodeText, Text: " tail"},
},
},
// no uid: the text is not indexable.
{
Type: document.BlockNodeParagraph,
Content: []document.Block{
{Type: document.BlockNodeText, Text: "orphan"},
},
},
// no text: nothing to index.
{Type: document.BlockNodeHorizontalRule, Attrs: document.Attributes{"uid": "hr1"}},
// a metric block's title is an attribute, not a text child.
{
Type: document.BlockNodeMetricBlock,
Attrs: document.Attributes{"uid": "m1", "title": "Pizza Fridays"},
},
// an untitled metric block has nothing to index.
{
Type: document.BlockNodeMetricBlock,
Attrs: document.Attributes{"uid": "m2", "title": ""},
},
{
Type: document.BlockNodeMetricBlock,
Attrs: document.Attributes{"uid": "m3"},
},
// a file block's name is an attribute, not a text child.
{
Type: document.BlockNodeFileBlock,
Attrs: document.Attributes{"uid": "f1", "name": "quarterly-report.pdf", "size": 2048},
},
// a file block still uploading has no name to index.
{
Type: document.BlockNodeFileBlock,
Attrs: document.Attributes{"uid": "f2"},
},
// an image block is described by its alt and title attributes.
{
Type: document.BlockNodeImageBlock,
Attrs: document.Attributes{"uid": "i1", "alt": "architecture diagram", "title": "Overview"},
},
{
Type: document.BlockNodeImageBlock,
Attrs: document.Attributes{"uid": "i2", "alt": "login flow"},
},
// an undescribed image has nothing to index.
{
Type: document.BlockNodeImageBlock,
Attrs: document.Attributes{"uid": "i3", "src": "/x.png"},
},
}
scope := stubScope()
res := make(map[string]Block)
for _, b := range blocks {
scope.entries(b, res)
}
assert.Equal(t, map[string]Block{
"p1": scope.Block("p1", "paragraph", "first"),
"li1": scope.Block("li1", "listItem", "nested"),
"p2": scope.Block("p2", "paragraph", "plain bold tail"),
"m1": scope.Block("m1", "metricBlock", "Pizza Fridays"),
"f1": scope.Block("f1", "fileBlock", "quarterly-report.pdf"),
"i1": scope.Block("i1", "imageBlock", "architecture diagram Overview"),
"i2": scope.Block("i2", "imageBlock", "login flow"),
}, res)
}
func Test_Entries(t *testing.T) {
t.Parallel()
doc := document.Document{
ID: xid.New(),
OrganizationID: "org-1",
BranchID: xid.New(),
BranchName: document.DefaultBranch,
DocumentName: "Runbook",
Content: document.RootBlock{
Type: document.BlockNodeDoc,
Content: []document.Block{
{
Type: document.BlockNodeParagraph,
Attrs: document.Attributes{"uid": "p1"},
Content: []document.Block{
{Type: document.BlockNodeText, Text: "hello"},
},
},
},
},
Default: true,
}
res := Entries(doc)
// the content block plus the synthetic document-name block.
require.Len(t, res, 2)
nameBlock, ok := res[doc.ID.String()]
require.True(t, ok)
assert.Equal(t, Block{
ID: doc.BranchID.String() + "-docname",
OrganizationID: "org-1",
DocumentID: doc.ID,
BranchID: doc.BranchID,
BranchName: document.DefaultBranch,
BranchDefault: true,
Type: "document",
Text: "Runbook",
}, nameBlock)
contentBlock, ok := res["p1"]
require.True(t, ok)
assert.Equal(t, doc.BranchID.String()+"-p1", contentBlock.ID)
assert.Equal(t, doc.BranchID, contentBlock.BranchID)
assert.True(t, contentBlock.BranchDefault)
assert.Equal(t, "hello", contentBlock.Text)
}

370
server/core/internal/search/index.go Normal file
View File

@ -0,0 +1,370 @@
// Package search indexes and searches documents through an embedded bleve
// index.
package search
import (
"context"
_ "embed"
"encoding/json"
"errors"
"fmt"
"log/slog"
"os"
"strconv"
"time"
"github.com/blevesearch/bleve/v2"
"github.com/oxynote/oxynote/server/core/internal/document"
"github.com/rs/xid"
)
//go:embed static/synonyms.json
var synonymsFile []byte
const (
// _buildSuffix is appended to the index path while a fresh index is
// being built next to it.
_buildSuffix = ".building"
// _buildPage is the number of branches one rebuild step reads.
_buildPage = 500
// _idsPage is the number of entry ids one lookup step reads.
_idsPage = 1000
// _writeTimeout bounds one batch of index writes.
_writeTimeout = 2 * time.Minute
)
// Index is the document index. Handlers search it concurrently; the
// search-job worker and the boot-time build are its only writers.
type Index struct {
log *slog.Logger
idx bleve.Index
}
// Open opens the index at path, or builds it from src when there is none
// or the one there was written by a binary with another mapping. The build
// happens next to path and is swapped in only once complete, so an
// interrupted build never leaves a half-written index behind.
func Open(ctx context.Context, log *slog.Logger, path string, src Source) (*Index, error) {
if path == "" {
return nil, errors.New("index path is empty")
}
if err := registerHighlighter(); err != nil {
return nil, err
}
log = log.With("component", "search-index")
idx, err := openCurrent(log, path)
if err != nil {
return nil, err
}
if idx == nil {
idx, err = build(ctx, log, path, src)
if err != nil {
return nil, err
}
}
return &Index{
log: log,
idx: idx,
}, nil
}
// openCurrent opens the index at path when it exists and carries the
// mapping version of this binary. It returns nil otherwise, which asks for
// a build; an index that cannot be opened counts as absent, since it is
// only ever a cache of the database.
func openCurrent(log *slog.Logger, path string) (bleve.Index, error) {
if _, err := os.Stat(path); err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, nil //nolint:nilnil // no index is a valid state that asks for a build
}
return nil, fmt.Errorf("checking index path: %w", err)
}
idx, err := bleve.Open(path)
if err != nil {
log.With("error", err).
Warn("cannot open the search index, rebuilding it")
return nil, nil //nolint:nilnil // an unreadable index asks for a build
}
version, err := idx.GetInternal([]byte(_mappingVersionKey))
if err != nil {
if cerr := idx.Close(); cerr != nil {
return nil, fmt.Errorf("closing index: %w", cerr)
}
return nil, fmt.Errorf("reading mapping version: %w", err)
}
if string(version) == _mappingVersion {
return idx, nil
}
log.With("index_version", string(version)).
With("binary_version", _mappingVersion).
Info("search index mapping is outdated, rebuilding it")
if err := idx.Close(); err != nil {
return nil, fmt.Errorf("closing index: %w", err)
}
return nil, nil //nolint:nilnil // an outdated index asks for a build
}
// build creates a fresh index from every branch in src at path.
func build(ctx context.Context, log *slog.Logger, path string, src Source) (bleve.Index, error) {
log.Info("building the search index")
tmp := path + _buildSuffix
if err := os.RemoveAll(tmp); err != nil {
return nil, fmt.Errorf("removing stale build: %w", err)
}
im, err := newIndexMapping()
if err != nil {
return nil, err
}
idx, err := bleve.New(tmp, im)
if err != nil {
return nil, fmt.Errorf("creating index: %w", err)
}
if err = fill(ctx, log, idx, src); err != nil {
if cerr := idx.Close(); cerr != nil {
err = errors.Join(err, fmt.Errorf("closing index: %w", cerr))
}
return nil, err
}
if err = idx.Close(); err != nil {
return nil, fmt.Errorf("closing built index: %w", err)
}
if err = os.RemoveAll(path); err != nil {
return nil, fmt.Errorf("removing previous index: %w", err)
}
if err = os.Rename(tmp, path); err != nil {
return nil, fmt.Errorf("moving built index into place: %w", err)
}
idx, err = bleve.Open(path)
if err != nil {
return nil, fmt.Errorf("opening built index: %w", err)
}
return idx, nil
}
// fill writes the synonyms, every branch's entries and finally the mapping
// version into a fresh index. The version goes last, so an index missing
// it is one whose build did not finish.
func fill(ctx context.Context, log *slog.Logger, idx bleve.Index, src Source) error {
if err := indexSynonyms(idx); err != nil {
return err
}
var (
after xid.ID
branches int
)
for {
docs, err := src.FetchDocumentBranchesAfter(ctx, after, _buildPage)
if err != nil {
return fmt.Errorf("fetching branches: %w", err)
}
batch := idx.NewBatch()
for _, doc := range docs {
for _, e := range Entries(doc) {
if err := batch.Index(e.ID, e.record()); err != nil {
return fmt.Errorf("indexing entry %s: %w", e.ID, err)
}
}
}
if err := idx.Batch(batch); err != nil {
return fmt.Errorf("writing batch: %w", err)
}
branches += len(docs)
if len(docs) < _buildPage {
break
}
after = docs[len(docs)-1].BranchID
}
if err := idx.SetInternal([]byte(_mappingVersionKey), []byte(_mappingVersion)); err != nil {
return fmt.Errorf("writing mapping version: %w", err)
}
log.With("branches", branches).
Info("built the search index")
return nil
}
// indexSynonyms loads the synonyms file into the index. Every entry is
// bidirectional: the key and each of its values expand to one another.
func indexSynonyms(idx bleve.Index) error {
var synonyms map[string][]string
if err := json.Unmarshal(synonymsFile, &synonyms); err != nil {
return fmt.Errorf("unmarshaling synonyms: %w", err)
}
batch := idx.NewBatch()
n := 0
for key, values := range synonyms {
n++
def := &bleve.SynonymDefinition{
Synonyms: append([]string{key}, values...),
}
if err := batch.IndexSynonym("synonym-"+strconv.Itoa(n), _synonymCollection, def); err != nil {
return fmt.Errorf("indexing synonym %q: %w", key, err)
}
}
if err := idx.Batch(batch); err != nil {
return fmt.Errorf("writing synonyms: %w", err)
}
return nil
}
// Close closes the index.
func (i *Index) Close() error {
return i.idx.Close()
}
// ReplaceBranch makes the branch's entries exactly the given ones: what is
// there and not in entries is deleted, the rest is written, all in one
// batch.
func (i *Index) ReplaceBranch(ctx context.Context, branchID xid.ID, entries map[string]Block) error {
ctx, cancel := context.WithTimeout(ctx, _writeTimeout)
defer cancel()
ids, err := i.ids(ctx, _fieldBranchID, branchID.String())
if err != nil {
return err
}
batch := i.idx.NewBatch()
// a block that survives an edit keeps its id, so it is deleted and
// indexed in the same batch. The batch is a map keyed by id where the
// last write wins, which is why the deletes go in first.
for _, id := range ids {
batch.Delete(id)
}
for _, e := range entries {
if err := batch.Index(e.ID, e.record()); err != nil {
return fmt.Errorf("indexing entry %s: %w", e.ID, err)
}
}
if err := i.idx.Batch(batch); err != nil {
return fmt.Errorf("replacing branch %s: %w", branchID, err)
}
return nil
}
// DeleteBranch removes every entry of the branch.
func (i *Index) DeleteBranch(ctx context.Context, branchID xid.ID) error {
return i.deleteByTerm(ctx, _fieldBranchID, branchID.String())
}
// DeleteDocument removes every entry of the document, on every branch.
func (i *Index) DeleteDocument(ctx context.Context, documentID xid.ID) error {
return i.deleteByTerm(ctx, _fieldDocumentID, documentID.String())
}
// DeleteOrganization removes every entry of the organization.
func (i *Index) DeleteOrganization(ctx context.Context, organizationID string) error {
return i.deleteByTerm(ctx, _fieldOrganizationID, organizationID)
}
// deleteByTerm removes every entry whose field holds the value.
func (i *Index) deleteByTerm(ctx context.Context, field, value string) error {
ctx, cancel := context.WithTimeout(ctx, _writeTimeout)
defer cancel()
ids, err := i.ids(ctx, field, value)
if err != nil {
return err
}
if len(ids) == 0 {
return nil
}
batch := i.idx.NewBatch()
for _, id := range ids {
batch.Delete(id)
}
if err := i.idx.Batch(batch); err != nil {
return fmt.Errorf("deleting by %s: %w", field, err)
}
return nil
}
// ids lists the ids of every entry whose field holds the value. bleve has
// no delete-by-query, so removals go through this list.
func (i *Index) ids(ctx context.Context, field, value string) ([]string, error) {
q := bleve.NewTermQuery(value)
q.SetField(field)
var ids []string
for from := 0; ; from += _idsPage {
req := bleve.NewSearchRequestOptions(q, _idsPage, from, false)
req.SortBy([]string{"_id"})
res, err := i.idx.SearchInContext(ctx, req)
if err != nil {
return nil, fmt.Errorf("listing entries by %s: %w", field, err)
}
for _, hit := range res.Hits {
ids = append(ids, hit.ID)
}
if len(res.Hits) < _idsPage {
return ids, nil
}
}
}
// Source is the database surface a build reads the branches from.
type Source interface {
// FetchDocumentBranchesAfter should return up to limit branches whose
// id sorts after the given one, joined with their document, in id
// order.
FetchDocumentBranchesAfter(ctx context.Context, after xid.ID, limit int) ([]document.Document, error)
}

543
server/core/internal/search/index_test.go Normal file
View File

@ -0,0 +1,543 @@
package search
import (
"context"
"log/slog"
"os"
"path/filepath"
"strconv"
"testing"
"github.com/oxynote/oxynote/server/core/internal/document"
"github.com/rs/xid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// stubSource serves fixed branches, in branch id order, page by page.
type stubSource struct {
docs []document.Document
err error
calls int
}
func (s *stubSource) FetchDocumentBranchesAfter(_ context.Context, after xid.ID, limit int) ([]document.Document, error) {
s.calls++
if s.err != nil {
return nil, s.err
}
res := []document.Document{}
for _, d := range s.docs {
if d.BranchID.String() > after.String() && len(res) < limit {
res = append(res, d)
}
}
return res, nil
}
// stubDocument builds a default-branch document of org-1 with one
// paragraph. Ids come from xid.New, which increase within a process, so
// documents built in sequence are already in branch id order.
func stubDocument(name, text string) document.Document {
return document.Document{
ID: xid.New(),
OrganizationID: "org-1",
BranchID: xid.New(),
BranchName: document.DefaultBranch,
DocumentName: name,
Content: document.RootBlock{
Type: document.BlockNodeDoc,
Content: []document.Block{
{
Type: document.BlockNodeParagraph,
Attrs: document.Attributes{"uid": "p1"},
Content: []document.Block{
{Type: document.BlockNodeText, Text: text},
},
},
},
},
Default: true,
}
}
// indexPath returns a fresh index path that does not exist yet.
func indexPath(t *testing.T) string {
t.Helper()
return filepath.Join(t.TempDir(), "index")
}
// openIndex opens or builds an index at path from src.
func openIndex(t *testing.T, path string, src Source) *Index {
t.Helper()
idx, err := Open(context.Background(), slog.New(slog.DiscardHandler), path, src)
require.NoError(t, err)
require.NotNil(t, idx)
return idx
}
// texts returns the text of every entry the query finds for org-1, in rank
// order.
func texts(t *testing.T, idx *Index, q string) []string {
t.Helper()
blocks, err := idx.SearchDocumentBlocks(context.Background(), "org-1", q, 50)
require.NoError(t, err)
res := make([]string, 0, len(blocks))
for _, b := range blocks {
res = append(res, b.Text)
}
return res
}
// entryIDs returns the ids of every entry under the field's value.
func entryIDs(t *testing.T, idx *Index, field, value string) []string {
t.Helper()
ids, err := idx.ids(context.Background(), field, value)
require.NoError(t, err)
return ids
}
func Test_Open(t *testing.T) {
t.Parallel()
type tcase struct {
Path string
Source *stubSource
Err error
Check func(t *testing.T, idx *Index, path string, src *stubSource)
}
cc := map[string]func(t *testing.T) tcase{
"Empty path": func(*testing.T) tcase {
return tcase{
Path: "",
Source: &stubSource{},
Err: assert.AnError,
}
},
"Error returned by Source.FetchDocumentBranchesAfter": func(t *testing.T) tcase {
path := indexPath(t)
return tcase{
Path: path,
Source: &stubSource{err: assert.AnError},
Err: assert.AnError,
Check: func(t *testing.T, _ *Index, path string, _ *stubSource) {
_, err := os.Stat(path)
assert.True(t, os.IsNotExist(err), "a failed build leaves no index behind")
},
}
},
"Missing index is built from the source": func(t *testing.T) tcase {
return tcase{
Path: indexPath(t),
Source: &stubSource{docs: []document.Document{stubDocument("Runbook", "restart the pods")}},
Check: func(t *testing.T, idx *Index, path string, src *stubSource) {
assert.Equal(t, 1, src.calls)
assert.Equal(t, []string{"restart the pods"}, texts(t, idx, "pods"))
version, err := idx.idx.GetInternal([]byte(_mappingVersionKey))
require.NoError(t, err)
assert.Equal(t, _mappingVersion, string(version))
_, err = os.Stat(path + _buildSuffix)
assert.True(t, os.IsNotExist(err), "the build directory is moved into place")
},
}
},
"Index with the current mapping version opens as is": func(t *testing.T) tcase {
path := indexPath(t)
prev := openIndex(t, path, &stubSource{docs: []document.Document{stubDocument("Runbook", "restart the pods")}})
require.NoError(t, prev.Close())
return tcase{
Path: path,
Source: &stubSource{docs: []document.Document{stubDocument("Other", "scale the cluster")}},
Check: func(t *testing.T, idx *Index, _ string, src *stubSource) {
assert.Equal(t, 0, src.calls, "the source is not consulted")
assert.Equal(t, []string{"restart the pods"}, texts(t, idx, "pods"))
assert.Empty(t, texts(t, idx, "cluster"))
},
}
},
"Index with another mapping version is rebuilt": func(t *testing.T) tcase {
path := indexPath(t)
prev := openIndex(t, path, &stubSource{docs: []document.Document{stubDocument("Runbook", "restart the pods")}})
require.NoError(t, prev.idx.SetInternal([]byte(_mappingVersionKey), []byte("0")))
require.NoError(t, prev.Close())
return tcase{
Path: path,
Source: &stubSource{docs: []document.Document{stubDocument("Other", "scale the cluster")}},
Check: func(t *testing.T, idx *Index, _ string, src *stubSource) {
assert.Equal(t, 1, src.calls)
assert.Empty(t, texts(t, idx, "pods"))
assert.Equal(t, []string{"scale the cluster"}, texts(t, idx, "cluster"))
},
}
},
"Index without a mapping version is rebuilt": func(t *testing.T) tcase {
path := indexPath(t)
prev := openIndex(t, path, &stubSource{})
require.NoError(t, prev.idx.DeleteInternal([]byte(_mappingVersionKey)))
require.NoError(t, prev.Close())
return tcase{
Path: path,
Source: &stubSource{},
Check: func(t *testing.T, _ *Index, _ string, src *stubSource) {
assert.Equal(t, 1, src.calls)
},
}
},
"Unreadable index is rebuilt": func(t *testing.T) tcase {
path := indexPath(t)
require.NoError(t, os.WriteFile(path, []byte("not an index"), 0o600))
return tcase{
Path: path,
Source: &stubSource{},
Check: func(t *testing.T, _ *Index, path string, src *stubSource) {
assert.Equal(t, 1, src.calls)
info, err := os.Stat(path)
require.NoError(t, err)
assert.True(t, info.IsDir())
},
}
},
"Stale build directory is replaced": func(t *testing.T) tcase {
path := indexPath(t)
require.NoError(t, os.MkdirAll(path+_buildSuffix, 0o750))
require.NoError(t, os.WriteFile(filepath.Join(path+_buildSuffix, "junk"), []byte("junk"), 0o600))
return tcase{
Path: path,
Source: &stubSource{},
Check: func(t *testing.T, _ *Index, path string, _ *stubSource) {
_, err := os.Stat(filepath.Join(path, "junk"))
assert.True(t, os.IsNotExist(err))
},
}
},
}
for cn, c := range cc {
t.Run(cn, func(t *testing.T) {
t.Parallel()
c := c(t)
idx, err := Open(context.Background(), slog.New(slog.DiscardHandler), c.Path, c.Source)
if c.Err != nil {
require.Error(t, err)
assert.Nil(t, idx)
} else {
require.NoError(t, err)
require.NotNil(t, idx)
defer func() { require.NoError(t, idx.Close()) }()
}
if c.Check != nil {
c.Check(t, idx, c.Path, c.Source)
}
})
}
}
func Test_openCurrent(t *testing.T) {
t.Parallel()
log := slog.New(slog.DiscardHandler)
// missing path
idx, err := openCurrent(log, indexPath(t))
require.NoError(t, err)
assert.Nil(t, idx)
// path that is not an index
path := indexPath(t)
require.NoError(t, os.WriteFile(path, []byte("not an index"), 0o600))
idx, err = openCurrent(log, path)
require.NoError(t, err)
assert.Nil(t, idx)
// current index
path = indexPath(t)
built := openIndex(t, path, &stubSource{})
require.NoError(t, built.Close())
idx, err = openCurrent(log, path)
require.NoError(t, err)
require.NotNil(t, idx)
require.NoError(t, idx.Close())
// outdated index
built = openIndex(t, path, &stubSource{})
require.NoError(t, built.idx.SetInternal([]byte(_mappingVersionKey), []byte("0")))
require.NoError(t, built.Close())
idx, err = openCurrent(log, path)
require.NoError(t, err)
assert.Nil(t, idx)
}
func Test_build(t *testing.T) {
t.Parallel()
log := slog.New(slog.DiscardHandler)
// error
path := indexPath(t)
idx, err := build(context.Background(), log, path, &stubSource{err: assert.AnError})
require.Error(t, err)
assert.Nil(t, idx)
_, err = os.Stat(path)
assert.True(t, os.IsNotExist(err))
// success, replacing whatever was at the path
require.NoError(t, os.WriteFile(path, []byte("previous"), 0o600))
idx, err = build(context.Background(), log, path, &stubSource{})
require.NoError(t, err)
require.NotNil(t, idx)
require.NoError(t, idx.Close())
info, err := os.Stat(path)
require.NoError(t, err)
assert.True(t, info.IsDir())
_, err = os.Stat(path + _buildSuffix)
assert.True(t, os.IsNotExist(err))
}
func Test_fill(t *testing.T) {
t.Parallel()
// error
idx := openIndex(t, indexPath(t), &stubSource{})
defer func() { require.NoError(t, idx.Close()) }()
require.Error(t, fill(context.Background(), slog.New(slog.DiscardHandler), idx.idx, &stubSource{err: assert.AnError}))
// success: every page is read until a short one, and the version is
// stamped last.
docs := make([]document.Document, 0, _buildPage+1)
for i := range _buildPage + 1 {
docs = append(docs, stubDocument("Doc "+strconv.Itoa(i), "page filler"))
}
src := &stubSource{docs: docs}
require.NoError(t, idx.idx.DeleteInternal([]byte(_mappingVersionKey)))
require.NoError(t, fill(context.Background(), slog.New(slog.DiscardHandler), idx.idx, src))
assert.Equal(t, 2, src.calls)
// one paragraph and one document-name entry per branch.
assert.Len(t, entryIDs(t, idx, _fieldOrganizationID, "org-1"), 2*(_buildPage+1))
version, err := idx.idx.GetInternal([]byte(_mappingVersionKey))
require.NoError(t, err)
assert.Equal(t, _mappingVersion, string(version))
}
func Test_indexSynonyms(t *testing.T) {
t.Parallel()
idx := openIndex(t, indexPath(t), &stubSource{})
// a build already loaded the synonyms; loading them again is an error
// only once the index is closed.
require.NoError(t, indexSynonyms(idx.idx))
require.NoError(t, idx.Close())
require.Error(t, indexSynonyms(idx.idx))
}
func Test_Index_Close(t *testing.T) {
t.Parallel()
idx := openIndex(t, indexPath(t), &stubSource{})
require.NoError(t, idx.Close())
}
func Test_Index_ReplaceBranch(t *testing.T) {
t.Parallel()
idx := openIndex(t, indexPath(t), &stubSource{})
defer func() { require.NoError(t, idx.Close()) }()
one, two := stubScope(), stubScope()
require.NoError(t, idx.ReplaceBranch(context.Background(), one.BranchID, map[string]Block{
"a": one.Block("a", "paragraph", "alpha"),
"b": one.Block("b", "paragraph", "beta"),
}))
require.NoError(t, idx.ReplaceBranch(context.Background(), two.BranchID, map[string]Block{
"c": two.Block("c", "paragraph", "gamma"),
}))
// the replacement updates, adds and removes, and leaves the other
// branch alone.
require.NoError(t, idx.ReplaceBranch(context.Background(), one.BranchID, map[string]Block{
"a": one.Block("a", "paragraph", "alpha changed"),
"d": one.Block("d", "paragraph", "delta"),
}))
assert.ElementsMatch(t, []string{one.BranchID.String() + "-a", one.BranchID.String() + "-d"}, entryIDs(t, idx, _fieldBranchID, one.BranchID.String()))
assert.Equal(t, []string{"alpha changed"}, texts(t, idx, "alpha"))
assert.Empty(t, texts(t, idx, "beta"))
assert.Equal(t, []string{"gamma"}, texts(t, idx, "gamma"))
// an entry without an id is refused before anything is written.
require.Error(t, idx.ReplaceBranch(context.Background(), one.BranchID, map[string]Block{
"e": {Text: "epsilon"},
}))
assert.Equal(t, []string{"alpha changed"}, texts(t, idx, "alpha"))
// a closed index fails the lookup.
require.NoError(t, idx.idx.Close())
require.Error(t, idx.ReplaceBranch(context.Background(), one.BranchID, nil))
idx.idx = openIndex(t, indexPath(t), &stubSource{}).idx
}
func Test_Index_DeleteBranch(t *testing.T) {
t.Parallel()
idx := openIndex(t, indexPath(t), &stubSource{})
defer func() { require.NoError(t, idx.Close()) }()
one, two := stubScope(), stubScope()
require.NoError(t, idx.ReplaceBranch(context.Background(), one.BranchID, map[string]Block{"a": one.Block("a", "paragraph", "alpha")}))
require.NoError(t, idx.ReplaceBranch(context.Background(), two.BranchID, map[string]Block{"b": two.Block("b", "paragraph", "beta")}))
require.NoError(t, idx.DeleteBranch(context.Background(), one.BranchID))
assert.Empty(t, entryIDs(t, idx, _fieldBranchID, one.BranchID.String()))
assert.Len(t, entryIDs(t, idx, _fieldBranchID, two.BranchID.String()), 1)
}
func Test_Index_DeleteDocument(t *testing.T) {
t.Parallel()
idx := openIndex(t, indexPath(t), &stubSource{})
defer func() { require.NoError(t, idx.Close()) }()
one, two := stubScope(), stubScope()
fork := one
fork.BranchID = xid.New()
require.NoError(t, idx.ReplaceBranch(context.Background(), one.BranchID, map[string]Block{"a": one.Block("a", "paragraph", "alpha")}))
require.NoError(t, idx.ReplaceBranch(context.Background(), fork.BranchID, map[string]Block{"a": fork.Block("a", "paragraph", "alpha")}))
require.NoError(t, idx.ReplaceBranch(context.Background(), two.BranchID, map[string]Block{"b": two.Block("b", "paragraph", "beta")}))
require.NoError(t, idx.DeleteDocument(context.Background(), one.DocumentID))
assert.Empty(t, entryIDs(t, idx, _fieldDocumentID, one.DocumentID.String()))
assert.Len(t, entryIDs(t, idx, _fieldDocumentID, two.DocumentID.String()), 1)
}
func Test_Index_DeleteOrganization(t *testing.T) {
t.Parallel()
idx := openIndex(t, indexPath(t), &stubSource{})
defer func() { require.NoError(t, idx.Close()) }()
one, two := stubScope(), stubScope()
two.OrganizationID = "org-2"
require.NoError(t, idx.ReplaceBranch(context.Background(), one.BranchID, map[string]Block{"a": one.Block("a", "paragraph", "alpha")}))
require.NoError(t, idx.ReplaceBranch(context.Background(), two.BranchID, map[string]Block{"b": two.Block("b", "paragraph", "beta")}))
require.NoError(t, idx.DeleteOrganization(context.Background(), "org-1"))
assert.Empty(t, entryIDs(t, idx, _fieldOrganizationID, "org-1"))
assert.Len(t, entryIDs(t, idx, _fieldOrganizationID, "org-2"), 1)
}
func Test_Index_deleteByTerm(t *testing.T) {
t.Parallel()
idx := openIndex(t, indexPath(t), &stubSource{})
defer func() { require.NoError(t, idx.Close()) }()
scope := stubScope()
require.NoError(t, idx.ReplaceBranch(context.Background(), scope.BranchID, map[string]Block{"a": scope.Block("a", "paragraph", "alpha")}))
// nothing under the value: no batch is written.
require.NoError(t, idx.deleteByTerm(context.Background(), _fieldBranchID, "missing"))
assert.Len(t, entryIDs(t, idx, _fieldBranchID, scope.BranchID.String()), 1)
// success
require.NoError(t, idx.deleteByTerm(context.Background(), _fieldBranchID, scope.BranchID.String()))
assert.Empty(t, entryIDs(t, idx, _fieldBranchID, scope.BranchID.String()))
// a closed index fails the lookup.
require.NoError(t, idx.idx.Close())
require.Error(t, idx.deleteByTerm(context.Background(), _fieldBranchID, scope.BranchID.String()))
idx.idx = openIndex(t, indexPath(t), &stubSource{}).idx
}
func Test_Index_ids(t *testing.T) {
t.Parallel()
idx := openIndex(t, indexPath(t), &stubSource{})
defer func() { require.NoError(t, idx.Close()) }()
scope := stubScope()
entries := make(map[string]Block, _idsPage+1)
for i := range _idsPage + 1 {
uid := "b" + strconv.Itoa(i)
entries[uid] = scope.Block(uid, "paragraph", "filler")
}
require.NoError(t, idx.ReplaceBranch(context.Background(), scope.BranchID, entries))
// every page is read, without duplicates.
ids := entryIDs(t, idx, _fieldBranchID, scope.BranchID.String())
assert.Len(t, ids, _idsPage+1)
seen := make(map[string]struct{}, len(ids))
for _, id := range ids {
seen[id] = struct{}{}
}
assert.Len(t, seen, _idsPage+1)
// a cancelled context fails the lookup.
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err := idx.ids(ctx, _fieldBranchID, scope.BranchID.String())
require.Error(t, err)
}

67
server/core/internal/search/jobs.go
View File

@ -1,38 +1,53 @@
package search
import "context"
import (
"github.com/guregu/null/v5"
"github.com/rs/xid"
)
// Jobs is the way into the document search-job queue: every enqueue in
// the codebase goes through it, so whether a deployment indexes at all
// is decided here rather than by each call site or the database layer.
type Jobs struct {
enabled bool
// Job names a scope of the index to bring back in line with the database:
// a branch, a whole document, or a whole organization. It carries no
// content, so applying it in any order or any number of times yields the
// same index.
type Job struct {
// ID is the unique identifier for the job.
ID int64 `db:"id"`
// Version counts the times the scope was queued while the job was
// pending. A job is deleted only at the version it was applied with,
// so a change committed during the application is picked up again.
Version int64 `db:"version"`
// OrganizationID is the organization the scope belongs to.
OrganizationID string `db:"organization_id"`
// DocumentID is the document the scope narrows to, if any.
DocumentID null.Value[xid.ID] `db:"document_id"`
// BranchID is the branch the scope narrows to, if any.
BranchID null.Value[xid.ID] `db:"branch_id"`
}
// NewJobs creates a fresh instance of Jobs. When enabled is false —
// search is not configured on this deployment — every enqueued job is
// dropped instead of piling up in a table nothing consumes; documents
// created meanwhile were never indexed, so there is nothing stale to
// clean up either.
func NewJobs(enabled bool) *Jobs {
return &Jobs{
enabled: enabled,
// BranchScope names one branch of a document.
func BranchScope(organizationID string, documentID, branchID xid.ID) Job {
return Job{
OrganizationID: organizationID,
DocumentID: null.ValueFrom(documentID),
BranchID: null.ValueFrom(branchID),
}
}
// Enqueue queues the diff through the given inserter, which is the
// caller's own database handle so the job commits atomically with the
// content change it describes.
func (j *Jobs) Enqueue(ctx context.Context, ins JobInserter, diff BlocksDifference) error {
if !j.enabled {
return nil
// DocumentScope names every branch of a document.
func DocumentScope(organizationID string, documentID xid.ID) Job {
return Job{
OrganizationID: organizationID,
DocumentID: null.ValueFrom(documentID),
}
return ins.InsertDocumentSearchJob(ctx, diff)
}
// JobInserter is the database surface Enqueue writes through.
type JobInserter interface {
// InsertDocumentSearchJob should insert the document search job.
InsertDocumentSearchJob(ctx context.Context, diff BlocksDifference) error
// OrganizationScope names every entry of an organization.
func OrganizationScope(organizationID string) Job {
return Job{
OrganizationID: organizationID,
}
}

57
server/core/internal/search/jobs_test.go
View File

@ -1,49 +1,40 @@
package search
import (
"context"
"testing"
"github.com/guregu/null/v5"
"github.com/rs/xid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// insertRecorder records InsertDocumentSearchJob calls for Jobs tests.
type insertRecorder struct {
diffs []BlocksDifference
err error
}
func (ir *insertRecorder) InsertDocumentSearchJob(_ context.Context, diff BlocksDifference) error {
ir.diffs = append(ir.diffs, diff)
return ir.err
}
func Test_NewJobs(t *testing.T) {
func Test_BranchScope(t *testing.T) {
t.Parallel()
j := NewJobs(true)
require.NotNil(t, j)
assert.True(t, j.enabled)
documentID, branchID := xid.New(), xid.New()
assert.Equal(t, Job{
OrganizationID: "org-1",
DocumentID: null.ValueFrom(documentID),
BranchID: null.ValueFrom(branchID),
}, BranchScope("org-1", documentID, branchID))
}
func Test_Jobs_Enqueue(t *testing.T) {
func Test_DocumentScope(t *testing.T) {
t.Parallel()
diff := BlocksDifference{
RemovedOrganizations: []string{"org1"},
}
documentID := xid.New()
// disabled drops the job without touching the inserter.
ir := &insertRecorder{}
require.NoError(t, NewJobs(false).Enqueue(context.Background(), ir, diff))
assert.Empty(t, ir.diffs)
// enabled writes through and propagates the inserter's error.
ir = &insertRecorder{err: assert.AnError}
err := NewJobs(true).Enqueue(context.Background(), ir, diff)
assert.Equal(t, assert.AnError, err)
require.Len(t, ir.diffs, 1)
assert.Equal(t, diff, ir.diffs[0])
assert.Equal(t, Job{
OrganizationID: "org-1",
DocumentID: null.ValueFrom(documentID),
}, DocumentScope("org-1", documentID))
}
func Test_OrganizationScope(t *testing.T) {
t.Parallel()
assert.Equal(t, Job{
OrganizationID: "org-1",
}, OrganizationScope("org-1"))
}

12
server/core/internal/search/main_test.go Normal file
View File

@ -0,0 +1,12 @@
package search
import (
"testing"
"github.com/oxynote/oxynote/server/core/pkg/testutil"
"go.uber.org/goleak"
)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m, testutil.IgnoreBleveWorkers())
}

221
server/core/internal/search/manager/0mock_db_test.go
View File

@ -5,9 +5,10 @@ package manager
import (
"context"
"sync"
"github.com/oxynote/oxynote/server/core/internal/document"
"github.com/oxynote/oxynote/server/core/internal/search"
"github.com/rs/xid"
"sync"
)
// Ensure, that DBMock does implement DB.
@ -20,11 +21,17 @@ var _ DB = &DBMock{}
//
// // make and configure a mocked DB
// mockedDB := &DBMock{
// DeleteDocumentSearchJobFunc: func(ctx context.Context, id int64) error {
// panic("mock out the DeleteDocumentSearchJob method")
// DeleteSearchJobFunc: func(ctx context.Context, id int64, version int64) error {
// panic("mock out the DeleteSearchJob method")
// },
// FetchDocumentSearchJobsFunc: func(ctx context.Context, offsetID int64, limit int64) ([]search.DocumentSearchJob, error) {
// panic("mock out the FetchDocumentSearchJobs method")
// FetchDocumentBranchesUnsafeFunc: func(ctx context.Context, docID xid.ID) ([]document.BranchSummary, error) {
// panic("mock out the FetchDocumentBranchesUnsafe method")
// },
// FetchDocumentUnsafeByBranchIDFunc: func(ctx context.Context, branchID xid.ID) (*document.Document, error) {
// panic("mock out the FetchDocumentUnsafeByBranchID method")
// },
// FetchSearchJobsFunc: func(ctx context.Context, offsetID int64, limit int64) ([]search.Job, error) {
// panic("mock out the FetchSearchJobs method")
// },
// }
//
@ -33,23 +40,45 @@ var _ DB = &DBMock{}
//
// }
type DBMock struct {
// DeleteDocumentSearchJobFunc mocks the DeleteDocumentSearchJob method.
DeleteDocumentSearchJobFunc func(ctx context.Context, id int64) error
// DeleteSearchJobFunc mocks the DeleteSearchJob method.
DeleteSearchJobFunc func(ctx context.Context, id int64, version int64) error
// FetchDocumentSearchJobsFunc mocks the FetchDocumentSearchJobs method.
FetchDocumentSearchJobsFunc func(ctx context.Context, offsetID int64, limit int64) ([]search.DocumentSearchJob, error)
// FetchDocumentBranchesUnsafeFunc mocks the FetchDocumentBranchesUnsafe method.
FetchDocumentBranchesUnsafeFunc func(ctx context.Context, docID xid.ID) ([]document.BranchSummary, error)
// FetchDocumentUnsafeByBranchIDFunc mocks the FetchDocumentUnsafeByBranchID method.
FetchDocumentUnsafeByBranchIDFunc func(ctx context.Context, branchID xid.ID) (*document.Document, error)
// FetchSearchJobsFunc mocks the FetchSearchJobs method.
FetchSearchJobsFunc func(ctx context.Context, offsetID int64, limit int64) ([]search.Job, error)
// calls tracks calls to the methods.
calls struct {
// DeleteDocumentSearchJob holds details about calls to the DeleteDocumentSearchJob method.
DeleteDocumentSearchJob []struct {
// DeleteSearchJob holds details about calls to the DeleteSearchJob method.
DeleteSearchJob []struct {
// Ctx is the ctx argument value.
Ctx context.Context
// ID is the id argument value.
ID int64
// Version is the version argument value.
Version int64
}
// FetchDocumentSearchJobs holds details about calls to the FetchDocumentSearchJobs method.
FetchDocumentSearchJobs []struct {
// FetchDocumentBranchesUnsafe holds details about calls to the FetchDocumentBranchesUnsafe method.
FetchDocumentBranchesUnsafe []struct {
// Ctx is the ctx argument value.
Ctx context.Context
// DocID is the docID argument value.
DocID xid.ID
}
// FetchDocumentUnsafeByBranchID holds details about calls to the FetchDocumentUnsafeByBranchID method.
FetchDocumentUnsafeByBranchID []struct {
// Ctx is the ctx argument value.
Ctx context.Context
// BranchID is the branchID argument value.
BranchID xid.ID
}
// FetchSearchJobs holds details about calls to the FetchSearchJobs method.
FetchSearchJobs []struct {
// Ctx is the ctx argument value.
Ctx context.Context
// OffsetID is the offsetID argument value.
@ -58,51 +87,137 @@ type DBMock struct {
Limit int64
}
}
lockDeleteDocumentSearchJob sync.RWMutex
lockFetchDocumentSearchJobs sync.RWMutex
lockDeleteSearchJob sync.RWMutex
lockFetchDocumentBranchesUnsafe sync.RWMutex
lockFetchDocumentUnsafeByBranchID sync.RWMutex
lockFetchSearchJobs sync.RWMutex
}
// DeleteDocumentSearchJob calls DeleteDocumentSearchJobFunc.
func (mock *DBMock) DeleteDocumentSearchJob(ctx context.Context, id int64) error {
// DeleteSearchJob calls DeleteSearchJobFunc.
func (mock *DBMock) DeleteSearchJob(ctx context.Context, id int64, version int64) error {
callInfo := struct {
Ctx context.Context
ID int64
Ctx context.Context
ID int64
Version int64
}{
Ctx: ctx,
ID: id,
Ctx: ctx,
ID: id,
Version: version,
}
mock.lockDeleteDocumentSearchJob.Lock()
mock.calls.DeleteDocumentSearchJob = append(mock.calls.DeleteDocumentSearchJob, callInfo)
mock.lockDeleteDocumentSearchJob.Unlock()
if mock.DeleteDocumentSearchJobFunc == nil {
mock.lockDeleteSearchJob.Lock()
mock.calls.DeleteSearchJob = append(mock.calls.DeleteSearchJob, callInfo)
mock.lockDeleteSearchJob.Unlock()
if mock.DeleteSearchJobFunc == nil {
var (
errOut error
)
return errOut
}
return mock.DeleteDocumentSearchJobFunc(ctx, id)
return mock.DeleteSearchJobFunc(ctx, id, version)
}
// DeleteDocumentSearchJobCalls gets all the calls that were made to DeleteDocumentSearchJob.
// DeleteSearchJobCalls gets all the calls that were made to DeleteSearchJob.
// Check the length with:
//
// len(mockedDB.DeleteDocumentSearchJobCalls())
func (mock *DBMock) DeleteDocumentSearchJobCalls() []struct {
Ctx context.Context
ID int64
// len(mockedDB.DeleteSearchJobCalls())
func (mock *DBMock) DeleteSearchJobCalls() []struct {
Ctx context.Context
ID int64
Version int64
} {
var calls []struct {
Ctx context.Context
ID int64
Ctx context.Context
ID int64
Version int64
}
mock.lockDeleteDocumentSearchJob.RLock()
calls = mock.calls.DeleteDocumentSearchJob
mock.lockDeleteDocumentSearchJob.RUnlock()
mock.lockDeleteSearchJob.RLock()
calls = mock.calls.DeleteSearchJob
mock.lockDeleteSearchJob.RUnlock()
return calls
}
// FetchDocumentSearchJobs calls FetchDocumentSearchJobsFunc.
func (mock *DBMock) FetchDocumentSearchJobs(ctx context.Context, offsetID int64, limit int64) ([]search.DocumentSearchJob, error) {
// FetchDocumentBranchesUnsafe calls FetchDocumentBranchesUnsafeFunc.
func (mock *DBMock) FetchDocumentBranchesUnsafe(ctx context.Context, docID xid.ID) ([]document.BranchSummary, error) {
callInfo := struct {
Ctx context.Context
DocID xid.ID
}{
Ctx: ctx,
DocID: docID,
}
mock.lockFetchDocumentBranchesUnsafe.Lock()
mock.calls.FetchDocumentBranchesUnsafe = append(mock.calls.FetchDocumentBranchesUnsafe, callInfo)
mock.lockFetchDocumentBranchesUnsafe.Unlock()
if mock.FetchDocumentBranchesUnsafeFunc == nil {
var (
out0 []document.BranchSummary
errOut error
)
return out0, errOut
}
return mock.FetchDocumentBranchesUnsafeFunc(ctx, docID)
}
// FetchDocumentBranchesUnsafeCalls gets all the calls that were made to FetchDocumentBranchesUnsafe.
// Check the length with:
//
// len(mockedDB.FetchDocumentBranchesUnsafeCalls())
func (mock *DBMock) FetchDocumentBranchesUnsafeCalls() []struct {
Ctx context.Context
DocID xid.ID
} {
var calls []struct {
Ctx context.Context
DocID xid.ID
}
mock.lockFetchDocumentBranchesUnsafe.RLock()
calls = mock.calls.FetchDocumentBranchesUnsafe
mock.lockFetchDocumentBranchesUnsafe.RUnlock()
return calls
}
// FetchDocumentUnsafeByBranchID calls FetchDocumentUnsafeByBranchIDFunc.
func (mock *DBMock) FetchDocumentUnsafeByBranchID(ctx context.Context, branchID xid.ID) (*document.Document, error) {
callInfo := struct {
Ctx context.Context
BranchID xid.ID
}{
Ctx: ctx,
BranchID: branchID,
}
mock.lockFetchDocumentUnsafeByBranchID.Lock()
mock.calls.FetchDocumentUnsafeByBranchID = append(mock.calls.FetchDocumentUnsafeByBranchID, callInfo)
mock.lockFetchDocumentUnsafeByBranchID.Unlock()
if mock.FetchDocumentUnsafeByBranchIDFunc == nil {
var (
out0 *document.Document
errOut error
)
return out0, errOut
}
return mock.FetchDocumentUnsafeByBranchIDFunc(ctx, branchID)
}
// FetchDocumentUnsafeByBranchIDCalls gets all the calls that were made to FetchDocumentUnsafeByBranchID.
// Check the length with:
//
// len(mockedDB.FetchDocumentUnsafeByBranchIDCalls())
func (mock *DBMock) FetchDocumentUnsafeByBranchIDCalls() []struct {
Ctx context.Context
BranchID xid.ID
} {
var calls []struct {
Ctx context.Context
BranchID xid.ID
}
mock.lockFetchDocumentUnsafeByBranchID.RLock()
calls = mock.calls.FetchDocumentUnsafeByBranchID
mock.lockFetchDocumentUnsafeByBranchID.RUnlock()
return calls
}
// FetchSearchJobs calls FetchSearchJobsFunc.
func (mock *DBMock) FetchSearchJobs(ctx context.Context, offsetID int64, limit int64) ([]search.Job, error) {
callInfo := struct {
Ctx context.Context
OffsetID int64
@ -112,24 +227,24 @@ func (mock *DBMock) FetchDocumentSearchJobs(ctx context.Context, offsetID int64,
OffsetID: offsetID,
Limit: limit,
}
mock.lockFetchDocumentSearchJobs.Lock()
mock.calls.FetchDocumentSearchJobs = append(mock.calls.FetchDocumentSearchJobs, callInfo)
mock.lockFetchDocumentSearchJobs.Unlock()
if mock.FetchDocumentSearchJobsFunc == nil {
mock.lockFetchSearchJobs.Lock()
mock.calls.FetchSearchJobs = append(mock.calls.FetchSearchJobs, callInfo)
mock.lockFetchSearchJobs.Unlock()
if mock.FetchSearchJobsFunc == nil {
var (
documentSearchJobsOut []search.DocumentSearchJob
errOut error
out0 []search.Job
errOut error
)
return documentSearchJobsOut, errOut
return out0, errOut
}
return mock.FetchDocumentSearchJobsFunc(ctx, offsetID, limit)
return mock.FetchSearchJobsFunc(ctx, offsetID, limit)
}
// FetchDocumentSearchJobsCalls gets all the calls that were made to FetchDocumentSearchJobs.
// FetchSearchJobsCalls gets all the calls that were made to FetchSearchJobs.
// Check the length with:
//
// len(mockedDB.FetchDocumentSearchJobsCalls())
func (mock *DBMock) FetchDocumentSearchJobsCalls() []struct {
// len(mockedDB.FetchSearchJobsCalls())
func (mock *DBMock) FetchSearchJobsCalls() []struct {
Ctx context.Context
OffsetID int64
Limit int64
@ -139,8 +254,8 @@ func (mock *DBMock) FetchDocumentSearchJobsCalls() []struct {
OffsetID int64
Limit int64
}
mock.lockFetchDocumentSearchJobs.RLock()
calls = mock.calls.FetchDocumentSearchJobs
mock.lockFetchDocumentSearchJobs.RUnlock()
mock.lockFetchSearchJobs.RLock()
calls = mock.calls.FetchSearchJobs
mock.lockFetchSearchJobs.RUnlock()
return calls
}

251
server/core/internal/search/manager/0mock_index_test.go Normal file
View File

@ -0,0 +1,251 @@
// Code generated by moq; DO NOT EDIT.
// github.com/matryer/moq
package manager
import (
"context"
"github.com/oxynote/oxynote/server/core/internal/search"
"github.com/rs/xid"
"sync"
)
// Ensure, that IndexMock does implement Index.
// If this is not the case, regenerate this file with moq.
var _ Index = &IndexMock{}
// IndexMock is a mock implementation of Index.
//
// func TestSomethingThatUsesIndex(t *testing.T) {
//
// // make and configure a mocked Index
// mockedIndex := &IndexMock{
// DeleteBranchFunc: func(ctx context.Context, branchID xid.ID) error {
// panic("mock out the DeleteBranch method")
// },
// DeleteDocumentFunc: func(ctx context.Context, documentID xid.ID) error {
// panic("mock out the DeleteDocument method")
// },
// DeleteOrganizationFunc: func(ctx context.Context, organizationID string) error {
// panic("mock out the DeleteOrganization method")
// },
// ReplaceBranchFunc: func(ctx context.Context, branchID xid.ID, entries map[string]search.Block) error {
// panic("mock out the ReplaceBranch method")
// },
// }
//
// // use mockedIndex in code that requires Index
// // and then make assertions.
//
// }
type IndexMock struct {
// DeleteBranchFunc mocks the DeleteBranch method.
DeleteBranchFunc func(ctx context.Context, branchID xid.ID) error
// DeleteDocumentFunc mocks the DeleteDocument method.
DeleteDocumentFunc func(ctx context.Context, documentID xid.ID) error
// DeleteOrganizationFunc mocks the DeleteOrganization method.
DeleteOrganizationFunc func(ctx context.Context, organizationID string) error
// ReplaceBranchFunc mocks the ReplaceBranch method.
ReplaceBranchFunc func(ctx context.Context, branchID xid.ID, entries map[string]search.Block) error
// calls tracks calls to the methods.
calls struct {
// DeleteBranch holds details about calls to the DeleteBranch method.
DeleteBranch []struct {
// Ctx is the ctx argument value.
Ctx context.Context
// BranchID is the branchID argument value.
BranchID xid.ID
}
// DeleteDocument holds details about calls to the DeleteDocument method.
DeleteDocument []struct {
// Ctx is the ctx argument value.
Ctx context.Context
// DocumentID is the documentID argument value.
DocumentID xid.ID
}
// DeleteOrganization holds details about calls to the DeleteOrganization method.
DeleteOrganization []struct {
// Ctx is the ctx argument value.
Ctx context.Context
// OrganizationID is the organizationID argument value.
OrganizationID string
}
// ReplaceBranch holds details about calls to the ReplaceBranch method.
ReplaceBranch []struct {
// Ctx is the ctx argument value.
Ctx context.Context
// BranchID is the branchID argument value.
BranchID xid.ID
// Entries is the entries argument value.
Entries map[string]search.Block
}
}
lockDeleteBranch sync.RWMutex
lockDeleteDocument sync.RWMutex
lockDeleteOrganization sync.RWMutex
lockReplaceBranch sync.RWMutex
}
// DeleteBranch calls DeleteBranchFunc.
func (mock *IndexMock) DeleteBranch(ctx context.Context, branchID xid.ID) error {
callInfo := struct {
Ctx context.Context
BranchID xid.ID
}{
Ctx: ctx,
BranchID: branchID,
}
mock.lockDeleteBranch.Lock()
mock.calls.DeleteBranch = append(mock.calls.DeleteBranch, callInfo)
mock.lockDeleteBranch.Unlock()
if mock.DeleteBranchFunc == nil {
var (
errOut error
)
return errOut
}
return mock.DeleteBranchFunc(ctx, branchID)
}
// DeleteBranchCalls gets all the calls that were made to DeleteBranch.
// Check the length with:
//
// len(mockedIndex.DeleteBranchCalls())
func (mock *IndexMock) DeleteBranchCalls() []struct {
Ctx context.Context
BranchID xid.ID
} {
var calls []struct {
Ctx context.Context
BranchID xid.ID
}
mock.lockDeleteBranch.RLock()
calls = mock.calls.DeleteBranch
mock.lockDeleteBranch.RUnlock()
return calls
}
// DeleteDocument calls DeleteDocumentFunc.
func (mock *IndexMock) DeleteDocument(ctx context.Context, documentID xid.ID) error {
callInfo := struct {
Ctx context.Context
DocumentID xid.ID
}{
Ctx: ctx,
DocumentID: documentID,
}
mock.lockDeleteDocument.Lock()
mock.calls.DeleteDocument = append(mock.calls.DeleteDocument, callInfo)
mock.lockDeleteDocument.Unlock()
if mock.DeleteDocumentFunc == nil {
var (
errOut error
)
return errOut
}
return mock.DeleteDocumentFunc(ctx, documentID)
}
// DeleteDocumentCalls gets all the calls that were made to DeleteDocument.
// Check the length with:
//
// len(mockedIndex.DeleteDocumentCalls())
func (mock *IndexMock) DeleteDocumentCalls() []struct {
Ctx context.Context
DocumentID xid.ID
} {
var calls []struct {
Ctx context.Context
DocumentID xid.ID
}
mock.lockDeleteDocument.RLock()
calls = mock.calls.DeleteDocument
mock.lockDeleteDocument.RUnlock()
return calls
}
// DeleteOrganization calls DeleteOrganizationFunc.
func (mock *IndexMock) DeleteOrganization(ctx context.Context, organizationID string) error {
callInfo := struct {
Ctx context.Context
OrganizationID string
}{
Ctx: ctx,
OrganizationID: organizationID,
}
mock.lockDeleteOrganization.Lock()
mock.calls.DeleteOrganization = append(mock.calls.DeleteOrganization, callInfo)
mock.lockDeleteOrganization.Unlock()
if mock.DeleteOrganizationFunc == nil {
var (
errOut error
)
return errOut
}
return mock.DeleteOrganizationFunc(ctx, organizationID)
}
// DeleteOrganizationCalls gets all the calls that were made to DeleteOrganization.
// Check the length with:
//
// len(mockedIndex.DeleteOrganizationCalls())
func (mock *IndexMock) DeleteOrganizationCalls() []struct {
Ctx context.Context
OrganizationID string
} {
var calls []struct {
Ctx context.Context
OrganizationID string
}
mock.lockDeleteOrganization.RLock()
calls = mock.calls.DeleteOrganization
mock.lockDeleteOrganization.RUnlock()
return calls
}
// ReplaceBranch calls ReplaceBranchFunc.
func (mock *IndexMock) ReplaceBranch(ctx context.Context, branchID xid.ID, entries map[string]search.Block) error {
callInfo := struct {
Ctx context.Context
BranchID xid.ID
Entries map[string]search.Block
}{
Ctx: ctx,
BranchID: branchID,
Entries: entries,
}
mock.lockReplaceBranch.Lock()
mock.calls.ReplaceBranch = append(mock.calls.ReplaceBranch, callInfo)
mock.lockReplaceBranch.Unlock()
if mock.ReplaceBranchFunc == nil {
var (
errOut error
)
return errOut
}
return mock.ReplaceBranchFunc(ctx, branchID, entries)
}
// ReplaceBranchCalls gets all the calls that were made to ReplaceBranch.
// Check the length with:
//
// len(mockedIndex.ReplaceBranchCalls())
func (mock *IndexMock) ReplaceBranchCalls() []struct {
Ctx context.Context
BranchID xid.ID
Entries map[string]search.Block
} {
var calls []struct {
Ctx context.Context
BranchID xid.ID
Entries map[string]search.Block
}
mock.lockReplaceBranch.RLock()
calls = mock.calls.ReplaceBranch
mock.lockReplaceBranch.RUnlock()
return calls
}

86
server/core/internal/search/manager/0mock_search_gateway_test.go
View File

@ -1,86 +0,0 @@
// Code generated by moq; DO NOT EDIT.
// github.com/matryer/moq
package manager
import (
"context"
"sync"
"github.com/oxynote/oxynote/server/core/internal/search"
)
// Ensure, that SearchGatewayMock does implement SearchGateway.
// If this is not the case, regenerate this file with moq.
var _ SearchGateway = &SearchGatewayMock{}
// SearchGatewayMock is a mock implementation of SearchGateway.
//
// func TestSomethingThatUsesSearchGateway(t *testing.T) {
//
// // make and configure a mocked SearchGateway
// mockedSearchGateway := &SearchGatewayMock{
// ReplaceDocumentBlocksFunc: func(ctx context.Context, bd search.BlocksDifference) error {
// panic("mock out the ReplaceDocumentBlocks method")
// },
// }
//
// // use mockedSearchGateway in code that requires SearchGateway
// // and then make assertions.
//
// }
type SearchGatewayMock struct {
// ReplaceDocumentBlocksFunc mocks the ReplaceDocumentBlocks method.
ReplaceDocumentBlocksFunc func(ctx context.Context, bd search.BlocksDifference) error
// calls tracks calls to the methods.
calls struct {
// ReplaceDocumentBlocks holds details about calls to the ReplaceDocumentBlocks method.
ReplaceDocumentBlocks []struct {
// Ctx is the ctx argument value.
Ctx context.Context
// Bd is the bd argument value.
Bd search.BlocksDifference
}
}
lockReplaceDocumentBlocks sync.RWMutex
}
// ReplaceDocumentBlocks calls ReplaceDocumentBlocksFunc.
func (mock *SearchGatewayMock) ReplaceDocumentBlocks(ctx context.Context, bd search.BlocksDifference) error {
callInfo := struct {
Ctx context.Context
Bd search.BlocksDifference
}{
Ctx: ctx,
Bd: bd,
}
mock.lockReplaceDocumentBlocks.Lock()
mock.calls.ReplaceDocumentBlocks = append(mock.calls.ReplaceDocumentBlocks, callInfo)
mock.lockReplaceDocumentBlocks.Unlock()
if mock.ReplaceDocumentBlocksFunc == nil {
var (
errOut error
)
return errOut
}
return mock.ReplaceDocumentBlocksFunc(ctx, bd)
}
// ReplaceDocumentBlocksCalls gets all the calls that were made to ReplaceDocumentBlocks.
// Check the length with:
//
// len(mockedSearchGateway.ReplaceDocumentBlocksCalls())
func (mock *SearchGatewayMock) ReplaceDocumentBlocksCalls() []struct {
Ctx context.Context
Bd search.BlocksDifference
} {
var calls []struct {
Ctx context.Context
Bd search.BlocksDifference
}
mock.lockReplaceDocumentBlocks.RLock()
calls = mock.calls.ReplaceDocumentBlocks
mock.lockReplaceDocumentBlocks.RUnlock()
return calls
}

296
server/core/internal/search/manager/manager.go
View File

@ -1,4 +1,4 @@
// Package manager runs the background document search-indexing jobs.
// Package manager runs the background search-indexing jobs.
package manager
import (
@ -7,219 +7,205 @@ import (
"log/slog"
"time"
"github.com/oxynote/oxynote/server/core/internal/document"
"github.com/oxynote/oxynote/server/core/internal/search"
"github.com/oxynote/oxynote/server/core/pkg/errutil"
"github.com/oxynote/oxynote/server/core/pkg/logutil"
"github.com/oxynote/oxynote/server/core/pkg/timeutil"
"github.com/rs/xid"
)
const (
// _processingBatch defines how many search jobs to process in one batch.
// _processingBatch defines how many search jobs to fetch in one batch.
_processingBatch = 100
// _processingInterval defines how often to process search jobs.
// _processingInterval defines how often queued jobs are processed
// when no nudge arrives, which covers jobs queued before a restart.
_processingInterval = time.Second * 10
)
// Manager manages document search index update jobs.
// Manager applies search jobs: for every queued scope it reads the
// current rows and makes the index match them.
type Manager struct {
log *slog.Logger
db DB
searchGateway SearchGateway
log *slog.Logger
db DB
index Index
pe *timeutil.PeriodicExec
}
// NewManager creates a new Manager with the given database and search gateway.
func NewManager(log *slog.Logger, db DB, searchGateway SearchGateway) *Manager {
return &Manager{
log: log.With("component", "document-search-job-manager"),
db: db,
searchGateway: searchGateway,
// NewManager creates a new Manager with the given database and index.
func NewManager(log *slog.Logger, db DB, index Index) *Manager {
m := &Manager{
log: log.With("component", "search-job-manager"),
db: db,
index: index,
}
m.pe = timeutil.NewPeriodicExec(
_processingInterval,
0,
m.run,
logutil.RecoveryValue(m.log, logutil.NewRecoveryPlan("recovered from a panic while processing search jobs")),
true,
)
return m
}
// Start begins the periodic processing of document search jobs.
// Start processes the queued jobs right away, then again on every trigger
// and on the periodic interval, until the context ends.
func (m *Manager) Start(ctx context.Context) {
m.log.Info("starting")
defer m.log.Info("stopped")
timeutil.NewPeriodicExec(
_processingInterval,
0,
func(ctx context.Context) {
if err := m.processJobs(ctx); err != nil {
m.log.With("error", err).
Error("processing document search jobs")
}
},
logutil.RecoveryValue(m.log, logutil.NewRecoveryPlan("recovered from a panic while processing document search jobs")),
true,
).Start(ctx)
m.pe.Start(ctx)
}
// processJobs processes document search jobs in a paginated manner. Failed
// jobs stay in the database and are retried on the next processing interval,
// together with every later job of the same document: a diff describes a
// change against the state its predecessors left behind, so applying one out
// of order resurrects blocks a newer diff removed.
// Trigger runs a pass right away. A job producer calls it once the job's
// transaction has committed, so the index follows a change without
// waiting for the periodic pass.
func (m *Manager) Trigger() {
m.pe.Trigger()
}
// run processes the queued jobs once, reporting a failed pass.
func (m *Manager) run(ctx context.Context) {
if err := m.processJobs(ctx); err != nil {
m.log.With("error", err).
Error("processing search jobs")
}
}
// processJobs applies the queued jobs in a paginated manner. A job that
// fails stays in the database for the next pass; the jobs after it are
// unaffected, since every job describes a scope's current state rather
// than a change against its predecessors.
func (m *Manager) processJobs(ctx context.Context) error {
var (
offsetID int64
held = newHoldSet()
)
var offsetID int64
for {
if err := ctx.Err(); err != nil {
return err
}
jobs, err := m.db.FetchDocumentSearchJobs(ctx, offsetID, _processingBatch)
jobs, err := m.db.FetchSearchJobs(ctx, offsetID, _processingBatch)
if err != nil {
return fmt.Errorf("fetching paginated document search jobs: %w", err)
return fmt.Errorf("fetching paginated search jobs: %w", err)
}
for _, job := range jobs {
offsetID = job.ID
documentIDs, global := jobScope(job.BlockDiff)
if held.blocks(documentIDs, global) {
held.hold(documentIDs, global)
if err := m.apply(ctx, job); err != nil {
m.log.With("job_id", job.ID).
With("error", err).
Error("applying search job")
continue
}
err := m.searchGateway.ReplaceDocumentBlocks(ctx, job.BlockDiff)
if err != nil {
// a job queued again meanwhile carries a newer version, so the
// delete leaves it for the next pass.
if err := m.db.DeleteSearchJob(ctx, job.ID, job.Version); err != nil {
m.log.With("job_id", job.ID).
With("error", err).
Error("processing document search job")
held.hold(documentIDs, global)
continue
}
err = m.db.DeleteDocumentSearchJob(ctx, job.ID)
if err != nil {
m.log.With("job_id", job.ID).
With("error", err).
Error("deleting document search job")
// the diff is applied but the job survives, so it runs again
// next interval — after anything newer, unless held.
held.hold(documentIDs, global)
Error("deleting search job")
}
}
if len(jobs) < _processingBatch {
break
return nil
}
}
}
// apply brings the job's scope in line with the database.
func (m *Manager) apply(ctx context.Context, job search.Job) error {
switch {
case job.BranchID.Valid:
return m.syncBranch(ctx, job.BranchID.V)
case job.DocumentID.Valid:
return m.syncDocument(ctx, job.DocumentID.V)
default:
// an organization is queued only by its teardown, which runs
// while its rows still exist, so there is nothing to compare
// against: the scope is always a removal.
return m.index.DeleteOrganization(ctx, job.OrganizationID)
}
}
// syncBranch replaces the branch's entries with what the database holds,
// or removes them when the branch is gone.
func (m *Manager) syncBranch(ctx context.Context, branchID xid.ID) error {
doc, err := m.db.FetchDocumentUnsafeByBranchID(ctx, branchID)
if err != nil {
if errutil.IsNotFound(err) {
return m.index.DeleteBranch(ctx, branchID)
}
return fmt.Errorf("fetching branch %s: %w", branchID, err)
}
return m.index.ReplaceBranch(ctx, branchID, search.Entries(*doc))
}
// syncDocument syncs every branch of the document, or removes the
// document's entries when it has none left.
func (m *Manager) syncDocument(ctx context.Context, documentID xid.ID) error {
branches, err := m.db.FetchDocumentBranchesUnsafe(ctx, documentID)
if err != nil {
return fmt.Errorf("fetching branches of document %s: %w", documentID, err)
}
if len(branches) == 0 {
return m.index.DeleteDocument(ctx, documentID)
}
for _, b := range branches {
if err := m.syncBranch(ctx, b.BranchID); err != nil {
return err
}
}
return nil
}
// jobScope reports the documents a difference touches. An organization
// removal is global: it clears the entries of every document the
// organization owns, which no per-document bound can enumerate.
func jobScope(bd search.BlocksDifference) (documentIDs []xid.ID, global bool) {
if len(bd.RemovedOrganizations) > 0 {
return nil, true
}
seen := make(map[xid.ID]struct{})
add := func(id xid.ID) {
if _, ok := seen[id]; ok {
return
}
seen[id] = struct{}{}
documentIDs = append(documentIDs, id)
}
for _, blocks := range [][]search.Block{bd.Added, bd.Updated, bd.Removed} {
for _, b := range blocks {
add(b.DocumentID)
}
}
for _, br := range bd.RemovedBranches {
add(br.DocumentID)
}
for _, id := range bd.RemovedDocuments {
add(id)
}
return documentIDs, false
}
// holdSet tracks the documents whose diffs have to wait for the next
// interval, in the order-preserving sense: once one diff of a document is
// held back, every later one is held with it.
type holdSet struct {
documentIDs map[xid.ID]struct{}
all bool
}
// newHoldSet creates an empty hold set.
func newHoldSet() *holdSet {
return &holdSet{documentIDs: make(map[xid.ID]struct{})}
}
// blocks reports whether a difference of the given scope has to wait.
func (h *holdSet) blocks(documentIDs []xid.ID, global bool) bool {
if h.all {
return true
}
// a global difference covers documents this run knows nothing about, so
// anything held at all can be one of them.
if global {
return len(h.documentIDs) > 0
}
for _, id := range documentIDs {
if _, ok := h.documentIDs[id]; ok {
return true
}
}
return false
}
// hold makes every later difference of the same scope wait too.
func (h *holdSet) hold(documentIDs []xid.ID, global bool) {
if global {
h.all = true
return
}
for _, id := range documentIDs {
h.documentIDs[id] = struct{}{}
}
}
// DB defines the database operations required by the Manager.
//
//go:generate ../../../scripts/codegen/mock -t internal DB db
type DB interface {
// FetchDocumentSearchJobs should retrieve a batch of document search
// jobs with IDs greater than the given offset ID, limited to the
// specified number of jobs.
FetchDocumentSearchJobs(ctx context.Context, offsetID, limit int64) ([]search.DocumentSearchJob, error)
// FetchSearchJobs should retrieve a batch of search jobs with IDs
// greater than the given offset ID, limited to the specified number
// of jobs.
FetchSearchJobs(ctx context.Context, offsetID, limit int64) ([]search.Job, error)
// DeleteDocumentSearchJob should delete the given document search job from the database.
DeleteDocumentSearchJob(ctx context.Context, id int64) error
// DeleteSearchJob should delete the search job when it is still at
// the given version.
DeleteSearchJob(ctx context.Context, id, version int64) error
// FetchDocumentUnsafeByBranchID should fetch the document joined
// against the branch, across organizations.
FetchDocumentUnsafeByBranchID(ctx context.Context, branchID xid.ID) (*document.Document, error)
// FetchDocumentBranchesUnsafe should fetch the document's branches,
// across organizations.
FetchDocumentBranchesUnsafe(ctx context.Context, docID xid.ID) ([]document.BranchSummary, error)
}
// SearchGateway defines the search operations required by the Manager.
// Index defines the index operations required by the Manager.
//
//go:generate ../../../scripts/codegen/mock -t internal SearchGateway search_gateway
type SearchGateway interface {
// ReplaceDocumentBlocks should replace document blocks based on the provided differences.
ReplaceDocumentBlocks(ctx context.Context, bd search.BlocksDifference) error
//go:generate ../../../scripts/codegen/mock -t internal Index index
type Index interface {
// ReplaceBranch should make the branch's entries exactly the given
// ones.
ReplaceBranch(ctx context.Context, branchID xid.ID, entries map[string]search.Block) error
// DeleteBranch should remove every entry of the branch.
DeleteBranch(ctx context.Context, branchID xid.ID) error
// DeleteDocument should remove every entry of the document.
DeleteDocument(ctx context.Context, documentID xid.ID) error
// DeleteOrganization should remove every entry of the organization.
DeleteOrganization(ctx context.Context, organizationID string) error
}

621
server/core/internal/search/manager/manager_test.go
View File

@ -2,85 +2,162 @@ package manager
import (
"context"
"database/sql"
"log/slog"
"strconv"
"path/filepath"
"testing"
"github.com/oxynote/oxynote/server/core/internal/document"
"github.com/oxynote/oxynote/server/core/internal/search"
"github.com/oxynote/oxynote/server/core/pkg/testutil"
"github.com/rs/xid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
)
// jobs builds n sequential search jobs with IDs starting at start.
func jobs(start, n int) []search.DocumentSearchJob {
jj := make([]search.DocumentSearchJob, 0, n)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m, testutil.IgnoreBleveWorkers())
}
// jobs builds n sequential branch-scope jobs with IDs starting at start.
func jobs(start, n int) []search.Job {
jj := make([]search.Job, 0, n)
for i := range n {
jj = append(jj, search.DocumentSearchJob{ID: int64(start + i)})
job := search.BranchScope("org-1", xid.New(), xid.New())
job.ID = int64(start + i)
job.Version = 1
jj = append(jj, job)
}
return jj
}
// docJob builds a job whose difference touches the given document.
func docJob(id int64, documentID xid.ID) search.DocumentSearchJob {
return search.DocumentSearchJob{
ID: id,
BlockDiff: search.BlocksDifference{
Added: []search.Block{{ID: "b" + strconv.FormatInt(id, 10), DocumentID: documentID}},
// stubDocument builds a default-branch document with one paragraph.
func stubDocument(branchID xid.ID) *document.Document {
return &document.Document{
ID: xid.New(),
OrganizationID: "org-1",
BranchID: branchID,
BranchName: document.DefaultBranch,
DocumentName: "Runbook",
Content: document.RootBlock{
Type: document.BlockNodeDoc,
Content: []document.Block{
{
Type: document.BlockNodeParagraph,
Attrs: document.Attributes{"uid": "p1"},
Content: []document.Block{
{Type: document.BlockNodeText, Text: "hello"},
},
},
},
},
Default: true,
}
}
// emptySource is a search.Source with no branches.
type emptySource struct{}
func (emptySource) FetchDocumentBranchesAfter(context.Context, xid.ID, int) ([]document.Document, error) {
return nil, nil
}
func Test_NewManager(t *testing.T) {
t.Parallel()
man := NewManager(slog.New(slog.DiscardHandler), &DBMock{}, &SearchGatewayMock{})
man := NewManager(slog.New(slog.DiscardHandler), &DBMock{}, &IndexMock{})
require.NotNil(t, man)
assert.NotNil(t, man.log)
assert.NotNil(t, man.db)
assert.NotNil(t, man.index)
assert.NotNil(t, man.pe)
}
func Test_Manager_Start(t *testing.T) {
t.Parallel()
fetched := make(chan struct{})
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
fetches := 0
var man *Manager
// the first pass runs at once; a trigger brings the second, which ends
// the run.
db := &DBMock{
FetchDocumentSearchJobsFunc: func(context.Context, int64, int64) ([]search.DocumentSearchJob, error) {
close(fetched)
cancel()
FetchSearchJobsFunc: func(context.Context, int64, int64) ([]search.Job, error) {
fetches++
switch fetches {
case 1:
man.Trigger()
case 2:
cancel()
}
return nil, nil
},
}
man = NewManager(slog.New(slog.DiscardHandler), db, &IndexMock{})
stopped := make(chan struct{})
go func() {
defer close(stopped)
NewManager(slog.New(slog.DiscardHandler), db, &SearchGatewayMock{}).Start(ctx)
man.Start(ctx)
}()
<-fetched
<-stopped
assert.Len(t, db.FetchDocumentSearchJobsCalls(), 1)
assert.Len(t, db.FetchSearchJobsCalls(), 2)
}
func Test_Manager_Trigger(t *testing.T) {
t.Parallel()
man := NewManager(slog.New(slog.DiscardHandler), &DBMock{}, &IndexMock{})
// repeated triggers never block; the executor's own tests cover the
// coalescing.
assert.NotPanics(t, func() {
man.Trigger()
man.Trigger()
})
}
func Test_Manager_run(t *testing.T) {
t.Parallel()
// an error is logged and swallowed
db := &DBMock{
FetchSearchJobsFunc: func(context.Context, int64, int64) ([]search.Job, error) {
return nil, assert.AnError
},
}
man := NewManager(slog.New(slog.DiscardHandler), db, &IndexMock{})
assert.NotPanics(t, func() { man.run(context.Background()) })
assert.Len(t, db.FetchSearchJobsCalls(), 1)
}
func Test_Manager_processJobs(t *testing.T) {
t.Parallel()
type check func(*testing.T, *DBMock, *SearchGatewayMock, error)
type check func(*testing.T, *DBMock, *IndexMock, error)
checks := func(cc ...check) []check { return cc }
hasError := func(expect bool) check {
return func(t *testing.T, _ *DBMock, _ *SearchGatewayMock, err error) {
return func(t *testing.T, _ *DBMock, _ *IndexMock, err error) {
if expect {
require.Error(t, err)
return
@ -91,29 +168,26 @@ func Test_Manager_processJobs(t *testing.T) {
}
wasReplaceCalled := func(count int) check {
return func(t *testing.T, _ *DBMock, gw *SearchGatewayMock, _ error) {
require.Len(t, gw.ReplaceDocumentBlocksCalls(), count)
return func(t *testing.T, _ *DBMock, idx *IndexMock, _ error) {
require.Len(t, idx.ReplaceBranchCalls(), count)
}
}
wasDeleteCalled := func(count int) check {
return func(t *testing.T, db *DBMock, _ *SearchGatewayMock, _ error) {
require.Len(t, db.DeleteDocumentSearchJobCalls(), count)
return func(t *testing.T, db *DBMock, _ *IndexMock, _ error) {
require.Len(t, db.DeleteSearchJobCalls(), count)
}
}
docA, docB := xid.New(), xid.New()
isCancelled := func() check {
return func(t *testing.T, _ *DBMock, _ *SearchGatewayMock, err error) {
assert.ErrorIs(t, err, context.Canceled)
return func(t *testing.T, _ *DBMock, _ *IndexMock, err error) {
assert.Equal(t, context.Canceled, err)
}
}
tests := map[string]struct {
Batches [][]search.DocumentSearchJob
cc := map[string]struct {
Batches [][]search.Job
CancelledContext bool
ReplaceErr error
FailJobs map[int64]bool
DeleteErr error
Checks []check
@ -125,143 +199,73 @@ func Test_Manager_processJobs(t *testing.T) {
wasReplaceCalled(0),
),
},
"Fetch failure is propagated": {
"Error returned by DB.FetchSearchJobs": {
Batches: nil,
Checks: checks(
hasError(true),
wasReplaceCalled(0),
),
},
"Jobs are processed and deleted": {
Batches: [][]search.DocumentSearchJob{jobs(1, 3)},
"Jobs are applied and deleted at their version": {
Batches: [][]search.Job{jobs(1, 3)},
Checks: checks(
hasError(false),
wasReplaceCalled(3),
wasDeleteCalled(3),
func(t *testing.T, db *DBMock, _ *IndexMock, _ error) {
for i, call := range db.DeleteSearchJobCalls() {
assert.Equal(t, int64(i+1), call.ID)
assert.Equal(t, int64(1), call.Version)
}
},
),
},
"Full batches keep paginating past the last job ID": {
Batches: [][]search.DocumentSearchJob{jobs(1, _processingBatch), jobs(101, 2)},
Batches: [][]search.Job{jobs(1, _processingBatch), jobs(101, 2)},
Checks: checks(
hasError(false),
wasReplaceCalled(_processingBatch+2),
wasDeleteCalled(_processingBatch+2),
func(t *testing.T, db *DBMock, _ *SearchGatewayMock, _ error) {
ff := db.FetchDocumentSearchJobsCalls()
func(t *testing.T, db *DBMock, _ *IndexMock, _ error) {
ff := db.FetchSearchJobsCalls()
require.Len(t, ff, 2)
assert.Equal(t, int64(0), ff[0].OffsetID)
assert.Equal(t, int64(_processingBatch), ff[1].OffsetID)
},
),
},
"Replace failure keeps the job for the next interval": {
Batches: [][]search.DocumentSearchJob{jobs(1, 2)},
ReplaceErr: assert.AnError,
"Failed job stays queued and the others still run": {
Batches: [][]search.Job{jobs(1, 3)},
FailJobs: map[int64]bool{2: true},
Checks: checks(
hasError(false),
wasReplaceCalled(2),
wasDeleteCalled(0),
wasReplaceCalled(3),
wasDeleteCalled(2),
func(t *testing.T, db *DBMock, _ *IndexMock, _ error) {
ff := db.DeleteSearchJobCalls()
require.Len(t, ff, 2)
assert.Equal(t, int64(1), ff[0].ID)
assert.Equal(t, int64(3), ff[1].ID)
},
),
},
"Persistently failing full batches advance and terminate": {
// the offset moves past failing jobs, so a wall of
// failures cannot refetch the same batch forever.
Batches: [][]search.DocumentSearchJob{
Batches: [][]search.Job{
jobs(1, _processingBatch),
jobs(101, _processingBatch),
nil,
},
ReplaceErr: assert.AnError,
FailJobs: map[int64]bool{-1: true},
Checks: checks(
hasError(false),
wasReplaceCalled(2*_processingBatch),
wasDeleteCalled(0),
),
},
// a diff is a delta against what the previous ones left, so replaying
// a failed one after a newer one would resurrect what the newer one
// removed — including the blocks of a document that no longer exists.
"A failed job holds the later diffs of its document": {
Batches: [][]search.DocumentSearchJob{{
docJob(1, docA),
docJob(2, docA),
docJob(3, docA),
}},
FailJobs: map[int64]bool{1: true},
Checks: checks(
hasError(false),
wasReplaceCalled(1),
wasDeleteCalled(0),
),
},
"A failure holds only its own document": {
Batches: [][]search.DocumentSearchJob{{
docJob(1, docA),
docJob(2, docB),
docJob(3, docA),
}},
FailJobs: map[int64]bool{1: true},
Checks: checks(
hasError(false),
wasReplaceCalled(2),
wasDeleteCalled(1),
func(t *testing.T, db *DBMock, _ *SearchGatewayMock, _ error) {
ff := db.DeleteDocumentSearchJobCalls()
require.Len(t, ff, 1)
assert.Equal(t, int64(2), ff[0].ID, "the other document keeps moving")
},
),
},
"A branch removal waits behind its document's failed diff": {
Batches: [][]search.DocumentSearchJob{{
docJob(1, docA),
{
ID: 2,
BlockDiff: search.BlocksDifference{
RemovedBranches: []search.BranchRemoval{{DocumentID: docA, BranchID: xid.New()}},
},
},
{
ID: 3,
BlockDiff: search.BlocksDifference{
RemovedBranches: []search.BranchRemoval{{DocumentID: docB, BranchID: xid.New()}},
},
},
}},
FailJobs: map[int64]bool{1: true},
Checks: checks(
hasError(false),
wasReplaceCalled(2),
wasDeleteCalled(1),
func(t *testing.T, db *DBMock, _ *SearchGatewayMock, _ error) {
ff := db.DeleteDocumentSearchJobCalls()
require.Len(t, ff, 1)
assert.Equal(t, int64(3), ff[0].ID, "the other document's branch removal keeps moving")
},
),
},
"An organization removal waits for anything held": {
Batches: [][]search.DocumentSearchJob{{
docJob(1, docA),
{
ID: 2,
BlockDiff: search.BlocksDifference{
RemovedOrganizations: []string{"org-1"},
},
},
docJob(3, docB),
}},
FailJobs: map[int64]bool{1: true},
Checks: checks(
hasError(false),
// the organization removal covers documents this run cannot
// enumerate, so everything after it waits as well.
wasReplaceCalled(1),
wasDeleteCalled(0),
),
},
"Delete failure still advances the offset": {
Batches: [][]search.DocumentSearchJob{
"Error returned by DB.DeleteSearchJob still advances": {
Batches: [][]search.Job{
jobs(1, _processingBatch),
nil,
},
@ -274,64 +278,377 @@ func Test_Manager_processJobs(t *testing.T) {
},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
for cn, c := range cc {
t.Run(cn, func(t *testing.T) {
t.Parallel()
fetches := 0
db := &DBMock{
FetchDocumentSearchJobsFunc: func(context.Context, int64, int64) ([]search.DocumentSearchJob, error) {
if fetches >= len(tc.Batches) {
if len(tc.Batches) == 0 {
FetchSearchJobsFunc: func(context.Context, int64, int64) ([]search.Job, error) {
if fetches >= len(c.Batches) {
if len(c.Batches) == 0 {
return nil, assert.AnError
}
t.Fatal("unexpected extra fetch: the pagination did not terminate")
}
batch := tc.Batches[fetches]
batch := c.Batches[fetches]
fetches++
return batch, nil
},
DeleteDocumentSearchJobFunc: func(context.Context, int64) error {
return tc.DeleteErr
DeleteSearchJobFunc: func(context.Context, int64, int64) error {
return c.DeleteErr
},
FetchDocumentUnsafeByBranchIDFunc: func(_ context.Context, branchID xid.ID) (*document.Document, error) {
return stubDocument(branchID), nil
},
}
applied := 0
gw := &SearchGatewayMock{
ReplaceDocumentBlocksFunc: func(context.Context, search.BlocksDifference) error {
idx := &IndexMock{
ReplaceBranchFunc: func(context.Context, xid.ID, map[string]search.Block) error {
applied++
if tc.FailJobs != nil {
// the fixture numbers jobs from one in fetch order.
if tc.FailJobs[int64(applied)] {
return assert.AnError
}
return nil
// the fixture numbers jobs from one in fetch order; a
// negative key fails every job.
if c.FailJobs[int64(applied)] || c.FailJobs[-1] {
return assert.AnError
}
return tc.ReplaceErr
return nil
},
}
man := NewManager(slog.New(slog.DiscardHandler), db, gw)
man := NewManager(slog.New(slog.DiscardHandler), db, idx)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if tc.CancelledContext {
if c.CancelledContext {
cancel()
}
err := man.processJobs(ctx)
for _, ch := range tc.Checks {
ch(t, db, gw, err)
for _, ch := range c.Checks {
ch(t, db, idx, err)
}
})
}
}
func Test_Manager_apply(t *testing.T) {
t.Parallel()
documentID, branchID := xid.New(), xid.New()
cc := map[string]struct {
Job search.Job
DB *DBMock
Err error
Call func(t *testing.T, idx *IndexMock)
}{
"Branch scope syncs the branch": {
Job: search.BranchScope("org-1", documentID, branchID),
DB: &DBMock{
FetchDocumentUnsafeByBranchIDFunc: func(_ context.Context, branchID xid.ID) (*document.Document, error) {
return stubDocument(branchID), nil
},
},
Call: func(t *testing.T, idx *IndexMock) {
require.Len(t, idx.ReplaceBranchCalls(), 1)
assert.Equal(t, branchID, idx.ReplaceBranchCalls()[0].BranchID)
},
},
"Document scope syncs the document": {
Job: search.DocumentScope("org-1", documentID),
DB: &DBMock{
FetchDocumentBranchesUnsafeFunc: func(context.Context, xid.ID) ([]document.BranchSummary, error) {
return nil, nil
},
},
Call: func(t *testing.T, idx *IndexMock) {
require.Len(t, idx.DeleteDocumentCalls(), 1)
assert.Equal(t, documentID, idx.DeleteDocumentCalls()[0].DocumentID)
},
},
"Organization scope removes the organization": {
Job: search.OrganizationScope("org-1"),
DB: &DBMock{},
Call: func(t *testing.T, idx *IndexMock) {
require.Len(t, idx.DeleteOrganizationCalls(), 1)
assert.Equal(t, "org-1", idx.DeleteOrganizationCalls()[0].OrganizationID)
},
},
"Error returned by Index.DeleteOrganization": {
Job: search.OrganizationScope("org-1"),
DB: &DBMock{},
Err: assert.AnError,
Call: func(t *testing.T, idx *IndexMock) {
require.Len(t, idx.DeleteOrganizationCalls(), 1)
},
},
}
for cn, c := range cc {
t.Run(cn, func(t *testing.T) {
t.Parallel()
idx := &IndexMock{
DeleteOrganizationFunc: func(context.Context, string) error {
return c.Err
},
}
man := NewManager(slog.New(slog.DiscardHandler), c.DB, idx)
err := man.apply(context.Background(), c.Job)
if c.Err != nil {
require.Error(t, err)
} else {
require.NoError(t, err)
}
c.Call(t, idx)
})
}
// applied against a real index, a set of jobs yields the same entries in
// every order, and applying one twice changes nothing.
t.Run("Jobs are order-independent and idempotent", func(t *testing.T) {
t.Parallel()
kept, gone := stubDocument(xid.New()), stubDocument(xid.New())
removed := xid.New()
db := &DBMock{
FetchDocumentUnsafeByBranchIDFunc: func(_ context.Context, branchID xid.ID) (*document.Document, error) {
if branchID == kept.BranchID {
return kept, nil
}
return nil, sql.ErrNoRows
},
FetchDocumentBranchesUnsafeFunc: func(context.Context, xid.ID) ([]document.BranchSummary, error) {
return nil, nil
},
}
set := []search.Job{
search.BranchScope("org-1", kept.ID, kept.BranchID),
search.BranchScope("org-1", gone.ID, gone.BranchID),
search.DocumentScope("org-1", removed),
search.OrganizationScope("org-2"),
}
orders := [][]int{
{0, 1, 2, 3},
{3, 2, 1, 0},
{1, 0, 3, 2},
{0, 0, 2, 1, 3, 1},
}
var expected []search.Block
for i, order := range orders {
idx, err := search.Open(context.Background(), slog.New(slog.DiscardHandler), filepath.Join(t.TempDir(), "index"), emptySource{})
require.NoError(t, err)
// the branches to remove start out present, so the removals
// have something to do.
require.NoError(t, idx.ReplaceBranch(context.Background(), gone.BranchID, search.Entries(*gone)))
man := NewManager(slog.New(slog.DiscardHandler), db, idx)
for _, j := range order {
require.NoError(t, man.apply(context.Background(), set[j]))
}
blocks, err := idx.SearchDocumentBlocks(context.Background(), "org-1", "hello runbook", 50)
require.NoError(t, err)
require.NoError(t, idx.Close())
if i == 0 {
expected = blocks
require.Len(t, expected, 2, "the kept branch's paragraph and name")
continue
}
assert.ElementsMatch(t, expected, blocks)
}
})
}
func Test_Manager_syncBranch(t *testing.T) {
t.Parallel()
branchID := xid.New()
cc := map[string]struct {
FetchErr error
ReplaceErr error
DeleteErr error
Err error
Replaced int
Deleted int
}{
"Error returned by DB.FetchDocumentUnsafeByBranchID": {
FetchErr: assert.AnError,
Err: assert.AnError,
},
"Missing branch is removed from the index": {
FetchErr: sql.ErrNoRows,
Deleted: 1,
},
"Error returned by Index.DeleteBranch": {
FetchErr: sql.ErrNoRows,
DeleteErr: assert.AnError,
Err: assert.AnError,
Deleted: 1,
},
"Error returned by Index.ReplaceBranch": {
ReplaceErr: assert.AnError,
Err: assert.AnError,
Replaced: 1,
},
"Branch entries are replaced": {
Replaced: 1,
},
}
for cn, c := range cc {
t.Run(cn, func(t *testing.T) {
t.Parallel()
doc := stubDocument(branchID)
db := &DBMock{
FetchDocumentUnsafeByBranchIDFunc: func(context.Context, xid.ID) (*document.Document, error) {
if c.FetchErr != nil {
return nil, c.FetchErr
}
return doc, nil
},
}
idx := &IndexMock{
ReplaceBranchFunc: func(context.Context, xid.ID, map[string]search.Block) error {
return c.ReplaceErr
},
DeleteBranchFunc: func(context.Context, xid.ID) error {
return c.DeleteErr
},
}
man := NewManager(slog.New(slog.DiscardHandler), db, idx)
err := man.syncBranch(context.Background(), branchID)
if c.Err != nil {
require.Error(t, err)
} else {
require.NoError(t, err)
}
require.Len(t, idx.ReplaceBranchCalls(), c.Replaced)
require.Len(t, idx.DeleteBranchCalls(), c.Deleted)
if c.Replaced == 1 {
call := idx.ReplaceBranchCalls()[0]
assert.Equal(t, branchID, call.BranchID)
assert.Equal(t, search.Entries(*doc), call.Entries)
}
if c.Deleted == 1 {
assert.Equal(t, branchID, idx.DeleteBranchCalls()[0].BranchID)
}
})
}
}
func Test_Manager_syncDocument(t *testing.T) {
t.Parallel()
documentID := xid.New()
branches := []document.BranchSummary{{BranchID: xid.New()}, {BranchID: xid.New()}}
cc := map[string]struct {
Branches []document.BranchSummary
FetchErr error
BranchErr error
DeleteErr error
Err error
Replaced int
DocDeleted int
}{
"Error returned by DB.FetchDocumentBranchesUnsafe": {
FetchErr: assert.AnError,
Err: assert.AnError,
},
"Document without branches is removed from the index": {
DocDeleted: 1,
},
"Error returned by Index.DeleteDocument": {
DeleteErr: assert.AnError,
Err: assert.AnError,
DocDeleted: 1,
},
"Every branch is synced": {
Branches: branches,
Replaced: 2,
},
"Error returned by a branch sync": {
Branches: branches,
BranchErr: assert.AnError,
Err: assert.AnError,
Replaced: 1,
},
}
for cn, c := range cc {
t.Run(cn, func(t *testing.T) {
t.Parallel()
db := &DBMock{
FetchDocumentBranchesUnsafeFunc: func(context.Context, xid.ID) ([]document.BranchSummary, error) {
return c.Branches, c.FetchErr
},
FetchDocumentUnsafeByBranchIDFunc: func(_ context.Context, branchID xid.ID) (*document.Document, error) {
return stubDocument(branchID), nil
},
}
idx := &IndexMock{
ReplaceBranchFunc: func(context.Context, xid.ID, map[string]search.Block) error {
return c.BranchErr
},
DeleteDocumentFunc: func(context.Context, xid.ID) error {
return c.DeleteErr
},
}
man := NewManager(slog.New(slog.DiscardHandler), db, idx)
err := man.syncDocument(context.Background(), documentID)
if c.Err != nil {
require.Error(t, err)
} else {
require.NoError(t, err)
}
require.Len(t, idx.ReplaceBranchCalls(), c.Replaced)
require.Len(t, idx.DeleteDocumentCalls(), c.DocDeleted)
if c.DocDeleted == 1 {
assert.Equal(t, documentID, idx.DeleteDocumentCalls()[0].DocumentID)
}
})
}

137
server/core/internal/search/mapping.go Normal file
View File

@ -0,0 +1,137 @@
package search
import (
"fmt"
"sync"
"github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
"github.com/blevesearch/bleve/v2/analysis/lang/en"
"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
"github.com/blevesearch/bleve/v2/mapping"
htmlFormatter "github.com/blevesearch/bleve/v2/search/highlight/format/html"
simpleFragmenter "github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple"
simpleHighlighter "github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple"
bleveIndex "github.com/blevesearch/bleve_index_api"
)
// _mappingVersion is stamped into every index built by this binary. Bump it
// whenever the analyzer, the fields, the synonyms file or anything else the
// index stores changes shape: an index carrying another version is rebuilt
// at boot.
const _mappingVersion = "1"
// _mappingVersionKey is the index-internal key holding the mapping version.
const _mappingVersionKey = "mapping-version"
// Entry field names, as they appear in Block's JSON.
const (
_fieldText = "text"
_fieldOrganizationID = "organizationId"
_fieldDocumentID = "documentId"
_fieldBranchID = "branchId"
_fieldBranchName = "branchName"
_fieldBranchDefault = "branchDefault"
_fieldType = "type"
)
const (
// _analyzerName is the analyzer the text field and the synonyms share.
_analyzerName = "text"
// _synonymSource is the synonym source the text field expands through.
_synonymSource = "synonyms"
// _synonymCollection is the collection the synonym definitions are
// indexed under.
_synonymCollection = "synonyms"
// _fragmenterName names the fragmenter cropping highlighted text.
_fragmenterName = "crop"
// _fragmentSize is the number of characters a highlighted fragment
// keeps around the best match.
_fragmentSize = 120
// _highlighterName names the highlighter wrapping matches in mark tags.
_highlighterName = "mark"
)
// _registerHighlighter guards the one-time registration of the highlighter
// in bleve's process-wide registry, which refuses a second definition.
var _registerHighlighter sync.Once
// newIndexMapping describes the entries: one text field analyzed for
// search, keyword fields to filter and boost on, and the branch name kept
// only for display.
func newIndexMapping() (*mapping.IndexMappingImpl, error) {
im := bleve.NewIndexMapping()
im.ScoringModel = bleveIndex.BM25Scoring
// no stop-word removal: phrases such as "not found" must stay
// searchable as written.
if err := im.AddCustomAnalyzer(_analyzerName, map[string]any{
"type": custom.Name,
"tokenizer": unicode.Name,
"token_filters": []any{lowercase.Name, en.SnowballStemmerName},
}); err != nil {
return nil, fmt.Errorf("adding analyzer: %w", err)
}
if err := im.AddSynonymSource(_synonymSource, map[string]any{
"collection": _synonymCollection,
"analyzer": _analyzerName,
}); err != nil {
return nil, fmt.Errorf("adding synonym source: %w", err)
}
text := bleve.NewTextFieldMapping()
text.Analyzer = _analyzerName
text.SynonymSource = _synonymSource
name := bleve.NewKeywordFieldMapping()
name.Index = false
dm := bleve.NewDocumentStaticMapping()
dm.AddFieldMappingsAt(_fieldText, text)
dm.AddFieldMappingsAt(_fieldOrganizationID, bleve.NewKeywordFieldMapping())
dm.AddFieldMappingsAt(_fieldDocumentID, bleve.NewKeywordFieldMapping())
dm.AddFieldMappingsAt(_fieldBranchID, bleve.NewKeywordFieldMapping())
dm.AddFieldMappingsAt(_fieldType, bleve.NewKeywordFieldMapping())
dm.AddFieldMappingsAt(_fieldBranchDefault, bleve.NewBooleanFieldMapping())
dm.AddFieldMappingsAt(_fieldBranchName, name)
im.DefaultMapping = dm
return im, nil
}
// registerHighlighter defines the highlighter search results are formatted
// with. bleve keeps highlighters in a process-wide registry, so this runs
// once per process rather than once per index.
func registerHighlighter() error {
var err error
_registerHighlighter.Do(func() {
_, err = bleve.Config.Cache.DefineFragmenter(_fragmenterName, map[string]any{
"type": simpleFragmenter.Name,
"size": float64(_fragmentSize),
})
if err != nil {
err = fmt.Errorf("defining fragmenter: %w", err)
return
}
_, err = bleve.Config.Cache.DefineHighlighter(_highlighterName, map[string]any{
"type": simpleHighlighter.Name,
"fragmenter": _fragmenterName,
"formatter": htmlFormatter.Name,
})
if err != nil {
err = fmt.Errorf("defining highlighter: %w", err)
}
})
return err
}

44
server/core/internal/search/mapping_test.go Normal file
View File

@ -0,0 +1,44 @@
package search
import (
"testing"
bleveIndex "github.com/blevesearch/bleve_index_api"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_newIndexMapping(t *testing.T) {
t.Parallel()
im, err := newIndexMapping()
require.NoError(t, err)
require.NotNil(t, im)
assert.Equal(t, bleveIndex.BM25Scoring, im.ScoringModel)
assert.Equal(t, _analyzerName, im.AnalyzerNameForPath(_fieldText))
assert.Equal(t, "keyword", im.AnalyzerNameForPath(_fieldOrganizationID))
assert.Equal(t, "keyword", im.AnalyzerNameForPath(_fieldBranchID))
assert.Equal(t, "keyword", im.AnalyzerNameForPath(_fieldDocumentID))
assert.Equal(t, "keyword", im.AnalyzerNameForPath(_fieldType))
// the analyzer lowercases and stems without dropping stop words.
tokens := im.AnalyzerNamed(_analyzerName).Analyze([]byte("Not Deploying"))
require.Len(t, tokens, 2)
assert.Equal(t, "not", string(tokens[0].Term))
assert.Equal(t, "deploy", string(tokens[1].Term))
// the branch name is kept for display only.
name := im.DefaultMapping.Properties[_fieldBranchName].Fields[0]
assert.False(t, name.Index)
assert.True(t, name.Store)
}
func Test_registerHighlighter(t *testing.T) {
t.Parallel()
// registration happens once per process, so a second call is a no-op
// rather than a duplicate-definition error.
require.NoError(t, registerHighlighter())
require.NoError(t, registerHighlighter())
}

232
server/core/internal/search/query.go Normal file
View File

@ -0,0 +1,232 @@
package search
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/blevesearch/bleve/v2"
bleveSearch "github.com/blevesearch/bleve/v2/search"
"github.com/blevesearch/bleve/v2/search/query"
"github.com/rs/xid"
)
const (
// _searchResultLimit caps the number of hits the search endpoint
// returns.
_searchResultLimit = 30
// _searchTimeout bounds one search request.
_searchTimeout = 10 * time.Second
// _phraseBoost lifts an entry holding the whole query as a phrase
// above one holding the same words scattered.
_phraseBoost = 2
// _documentNameBoost lifts an entry carrying a document name.
_documentNameBoost = 3
// _headingBoost lifts a heading above the other block types.
_headingBoost = 2
// _defaultBranchBoost lifts an entry on the document's default branch
// above the same entry on a fork.
_defaultBranchBoost = 1.5
// _maxQueryTerms is how many terms of a query are searched; the rest
// are dropped. Every term costs a dictionary walk for its fuzzy and
// prefix candidates.
_maxQueryTerms = 16
// _maxTermSearchers caps the terms one search expands to across its
// whole query tree, fuzzy and prefix candidates included, so a query
// over a large vocabulary fails instead of pinning the process.
_maxTermSearchers = 1024
)
// SearchDocuments searches the organization's entries and returns them as
// the search endpoint's JSON: the matched terms wrapped in mark tags and
// the text cropped to a fragment around the best match.
func (i *Index) SearchDocuments(ctx context.Context, organizationID, q string) ([]byte, error) {
blocks, err := i.search(ctx, organizationID, q, _searchResultLimit, true)
if err != nil {
return nil, err
}
data, err := json.Marshal(blocks)
if err != nil {
return nil, fmt.Errorf("marshaling search results: %w", err)
}
return data, nil
}
// SearchDocumentBlocks returns the raw matching blocks for the query,
// scoped to the organization. Unlike SearchDocuments it returns plain
// block values (no highlight markup or cropping) so callers, the AI
// assistant's search tool, can consume the text directly.
func (i *Index) SearchDocumentBlocks(ctx context.Context, organizationID, q string, limit int) ([]Block, error) {
return i.search(ctx, organizationID, q, limit, false)
}
// search runs the query and decodes the hits.
func (i *Index) search(ctx context.Context, organizationID, q string, limit int, highlight bool) ([]Block, error) {
ctx, cancel := context.WithTimeout(ctx, _searchTimeout)
defer cancel()
ctx = context.WithValue(ctx, bleveSearch.MaxTermSearchersKey, _maxTermSearchers)
req := bleve.NewSearchRequestOptions(i.query(organizationID, q), limit, 0, false)
req.Fields = []string{"*"}
if highlight {
req.Highlight = bleve.NewHighlightWithStyle(_highlighterName)
req.Highlight.AddField(_fieldText)
}
res, err := i.idx.SearchInContext(ctx, req)
if err != nil {
return nil, fmt.Errorf("searching documents: %w", err)
}
blocks := make([]Block, 0, len(res.Hits))
for _, hit := range res.Hits {
b, err := i.decodeHit(hit.ID, hit.Fields)
if err != nil {
return nil, err
}
// a hit reached only through a synonym has no matched term of its
// own to fragment around, so it keeps its full text.
if fragments := hit.Fragments[_fieldText]; highlight && len(fragments) != 0 {
b.Text = fragments[0]
}
blocks = append(blocks, b)
}
return blocks, nil
}
// query composes the search: an entry must belong to the organization and
// match the text, as the whole phrase or by its terms; a document name, a
// heading and the default branch then add score on top.
func (i *Index) query(organizationID, q string) query.Query {
org := bleve.NewTermQuery(organizationID)
org.SetField(_fieldOrganizationID)
var text query.Query = bleve.NewMatchNoneQuery()
// the analyzer is the field's own, so the terms run over the stemmed
// vocabulary the index holds.
if terms := i.terms(q); len(terms) != 0 {
phrase := bleve.NewPhraseQuery(terms, _fieldText)
phrase.SetBoost(_phraseBoost)
text = bleve.NewDisjunctionQuery(phrase, termsQuery(terms))
}
name := bleve.NewTermQuery("document")
name.SetField(_fieldType)
name.SetBoost(_documentNameBoost)
heading := bleve.NewTermQuery("heading")
heading.SetField(_fieldType)
heading.SetBoost(_headingBoost)
def := bleve.NewBoolFieldQuery(true)
def.SetField(_fieldBranchDefault)
def.SetBoost(_defaultBranchBoost)
bq := bleve.NewBooleanQuery()
bq.AddMust(org, text)
bq.AddShould(name, heading, def)
return bq
}
// terms analyzes the query into at most _maxQueryTerms index terms.
func (i *Index) terms(q string) []string {
tokens := i.idx.Mapping().AnalyzerNamed(_analyzerName).Analyze([]byte(q))
terms := make([]string, 0, min(len(tokens), _maxQueryTerms))
for _, token := range tokens[:min(len(tokens), _maxQueryTerms)] {
terms = append(terms, string(token.Term))
}
return terms
}
// termsQuery matches the terms with automatic fuzziness, the last one also
// as a prefix since it is what the user is still typing. All terms but one
// have to match: a lone unknown word does not empty the results, while a
// common word inside otherwise unknown ones does not fill them.
func termsQuery(terms []string) query.Query {
dq := bleve.NewDisjunctionQuery()
for n, term := range terms {
fuzzy := bleve.NewFuzzyQuery(term)
fuzzy.SetField(_fieldText)
fuzzy.SetAutoFuzziness(true)
if n != len(terms)-1 {
dq.AddQuery(fuzzy)
continue
}
prefix := bleve.NewPrefixQuery(term)
prefix.SetField(_fieldText)
dq.AddQuery(bleve.NewDisjunctionQuery(fuzzy, prefix))
}
dq.SetMin(float64(max(len(terms)-1, 1)))
return dq
}
// decodeHit rebuilds an entry from a hit's stored fields. Every field is
// stored with the type record gives it, so one of another type is a bug;
// it is logged and left at its zero value rather than failing the search.
func (i *Index) decodeHit(id string, fields map[string]any) (Block, error) {
str := func(field string) string {
s, ok := fields[field].(string)
if !ok {
i.log.With("entry_id", id).
With("field", field).
Warn("search entry field is not a string")
}
return s
}
documentID, err := xid.FromString(str(_fieldDocumentID))
if err != nil {
return Block{}, fmt.Errorf("decoding document id of entry %s: %w", id, err)
}
branchID, err := xid.FromString(str(_fieldBranchID))
if err != nil {
return Block{}, fmt.Errorf("decoding branch id of entry %s: %w", id, err)
}
def, ok := fields[_fieldBranchDefault].(bool)
if !ok {
i.log.With("entry_id", id).
With("field", _fieldBranchDefault).
Warn("search entry field is not a boolean")
}
return Block{
ID: id,
OrganizationID: str(_fieldOrganizationID),
DocumentID: documentID,
BranchID: branchID,
BranchName: str(_fieldBranchName),
BranchDefault: def,
Type: str(_fieldType),
Text: str(_fieldText),
}, nil
}

544
server/core/internal/search/query_test.go Normal file
View File

@ -0,0 +1,544 @@
package search
import (
"bytes"
"context"
"encoding/json"
"log/slog"
"strings"
"testing"
"github.com/blevesearch/bleve/v2/search/query"
"github.com/rs/xid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// _longText holds a match well past the fragment size on both sides.
const _longText = "the migration guide starts with the prerequisites and then walks through every step of the upgrade, " +
"including the zebra pattern used for rolling out schema changes without downtime, " +
"before closing with the rollback procedure and the checks to run afterwards"
// seedIndex fills an index with the entries the ranking cases rely on:
// a default branch and a fork of one document in org-1, and one document
// in org-2. It returns the index and the default branch's scope.
func seedIndex(t *testing.T) (*Index, Scope) {
t.Helper()
idx := openIndex(t, indexPath(t), &stubSource{})
t.Cleanup(func() { require.NoError(t, idx.Close()) })
main := Scope{
OrganizationID: "org-1",
DocumentID: xid.New(),
BranchID: xid.New(),
BranchName: "main",
BranchDefault: true,
}
fork := main
fork.BranchID = xid.New()
fork.BranchName = "draft"
fork.BranchDefault = false
other := Scope{
OrganizationID: "org-2",
DocumentID: xid.New(),
BranchID: xid.New(),
BranchName: "main",
BranchDefault: true,
}
require.NoError(t, idx.ReplaceBranch(context.Background(), main.BranchID, map[string]Block{
"docname": main.Block("docname", "document", "Release Checklist"),
"h1": main.Block("h1", "heading", "Release Checklist"),
"p1": main.Block("p1", "paragraph", "Release Checklist"),
"p2": main.Block("p2", "paragraph", "we deploy the service to production every friday"),
"p3": main.Block("p3", "paragraph", "production traffic hits the service after we deploy on monday"),
"p4": main.Block("p4", "paragraph", "alpha beta"),
"p5": main.Block("p5", "paragraph", "alpha only"),
"p6": main.Block("p6", "paragraph", "kubernetes clusters"),
"p7": main.Block("p7", "paragraph", "the endpoint answers 404 not found"),
"p8": main.Block("p8", "paragraph", "run an a11y audit before launch"),
"p9": main.Block("p9", "paragraph", "accessibility matters"),
"p10": main.Block("p10", "paragraph", _longText),
}))
require.NoError(t, idx.ReplaceBranch(context.Background(), fork.BranchID, map[string]Block{
"p1": fork.Block("p1", "paragraph", "Release Checklist"),
}))
require.NoError(t, idx.ReplaceBranch(context.Background(), other.BranchID, map[string]Block{
"p1": other.Block("p1", "paragraph", "Release Checklist"),
}))
return idx, main
}
func Test_Index_SearchDocuments(t *testing.T) {
t.Parallel()
idx, main := seedIndex(t)
data, err := idx.SearchDocuments(context.Background(), "org-1", "kubernetes")
require.NoError(t, err)
exp, err := json.Marshal([]Block{
{
ID: main.BranchID.String() + "-p6",
OrganizationID: "org-1",
DocumentID: main.DocumentID,
BranchID: main.BranchID,
BranchName: "main",
BranchDefault: true,
Type: "paragraph",
Text: "<mark>kubernetes</mark> clusters",
},
})
require.NoError(t, err)
assert.JSONEq(t, string(exp), string(data))
// error
ctx, cancel := context.WithCancel(context.Background())
cancel()
data, err = idx.SearchDocuments(ctx, "org-1", "kubernetes")
require.Error(t, err)
assert.Nil(t, data)
}
func Test_Index_SearchDocumentBlocks(t *testing.T) {
t.Parallel()
idx, main := seedIndex(t)
blocks, err := idx.SearchDocumentBlocks(context.Background(), "org-1", "kubernetes", 10)
require.NoError(t, err)
assert.Equal(t, []Block{
{
ID: main.BranchID.String() + "-p6",
OrganizationID: "org-1",
DocumentID: main.DocumentID,
BranchID: main.BranchID,
BranchName: "main",
BranchDefault: true,
Type: "paragraph",
Text: "kubernetes clusters",
},
}, blocks)
// the limit caps the hits.
blocks, err = idx.SearchDocumentBlocks(context.Background(), "org-1", "release checklist", 2)
require.NoError(t, err)
assert.Len(t, blocks, 2)
// error
ctx, cancel := context.WithCancel(context.Background())
cancel()
blocks, err = idx.SearchDocumentBlocks(ctx, "org-1", "kubernetes", 10)
require.Error(t, err)
assert.Nil(t, blocks)
}
func Test_Index_search(t *testing.T) {
t.Parallel()
idx, main := seedIndex(t)
cc := map[string]struct {
OrganizationID string
Query string
Limit int
Highlight bool
Check func(t *testing.T, blocks []Block)
}{
"Stemming matches another form of the word": {
OrganizationID: "org-1",
Query: "deploying",
Limit: 10,
Check: func(t *testing.T, blocks []Block) {
assert.ElementsMatch(t, []string{
"we deploy the service to production every friday",
"production traffic hits the service after we deploy on monday",
}, blockTexts(blocks))
},
},
"Stop words are searchable": {
OrganizationID: "org-1",
Query: "not found",
Limit: 10,
Check: func(t *testing.T, blocks []Block) {
require.NotEmpty(t, blocks)
assert.Equal(t, "the endpoint answers 404 not found", blocks[0].Text)
},
},
"Typo within the automatic fuzziness matches": {
OrganizationID: "org-1",
Query: "kubernets",
Limit: 10,
Check: func(t *testing.T, blocks []Block) {
assert.Equal(t, []string{"kubernetes clusters"}, blockTexts(blocks))
},
},
"Last term matches as a prefix": {
OrganizationID: "org-1",
Query: "kuber",
Limit: 10,
Check: func(t *testing.T, blocks []Block) {
assert.Equal(t, []string{"kubernetes clusters"}, blockTexts(blocks))
},
},
"Phrase ranks above scattered words": {
OrganizationID: "org-1",
Query: "deploy the service",
Limit: 10,
Check: func(t *testing.T, blocks []Block) {
// "the" alone matches other entries too; they rank below both.
require.GreaterOrEqual(t, len(blocks), 2)
assert.Equal(t, "we deploy the service to production every friday", blocks[0].Text)
assert.Equal(t, "production traffic hits the service after we deploy on monday", blocks[1].Text)
},
},
"All terms rank above a subset, which is still returned": {
OrganizationID: "org-1",
Query: "alpha beta",
Limit: 10,
Check: func(t *testing.T, blocks []Block) {
require.Len(t, blocks, 2)
assert.Equal(t, "alpha beta", blocks[0].Text)
assert.Equal(t, "alpha only", blocks[1].Text)
},
},
"Synonym expands the query": {
OrganizationID: "org-1",
Query: "accessibility",
Limit: 10,
Check: func(t *testing.T, blocks []Block) {
assert.ElementsMatch(t, []string{
"run an a11y audit before launch",
"accessibility matters",
}, blockTexts(blocks))
},
},
"Synonym expands the query in the other direction": {
OrganizationID: "org-1",
Query: "a11y",
Limit: 10,
Check: func(t *testing.T, blocks []Block) {
assert.ElementsMatch(t, []string{
"run an a11y audit before launch",
"accessibility matters",
}, blockTexts(blocks))
},
},
"Document name, heading and default branch rank first": {
OrganizationID: "org-1",
Query: "release checklist",
Limit: 10,
Check: func(t *testing.T, blocks []Block) {
require.Len(t, blocks, 4)
assert.Equal(t, "document", blocks[0].Type)
assert.Equal(t, "heading", blocks[1].Type)
assert.Equal(t, "paragraph", blocks[2].Type)
assert.True(t, blocks[2].BranchDefault)
assert.Equal(t, "paragraph", blocks[3].Type)
assert.False(t, blocks[3].BranchDefault)
},
},
"Entries of another organization are never returned": {
OrganizationID: "org-2",
Query: "release checklist",
Limit: 10,
Check: func(t *testing.T, blocks []Block) {
require.Len(t, blocks, 1)
assert.Equal(t, "org-2", blocks[0].OrganizationID)
},
},
"Unknown organization finds nothing": {
OrganizationID: "org-3",
Query: "release checklist",
Limit: 10,
Check: func(t *testing.T, blocks []Block) {
assert.Empty(t, blocks)
},
},
"Limit caps the hits": {
OrganizationID: "org-1",
Query: "release checklist",
Limit: 1,
Check: func(t *testing.T, blocks []Block) {
assert.Len(t, blocks, 1)
},
},
"Highlight marks the match inside a fragment": {
OrganizationID: "org-1",
Query: "zebra",
Limit: 10,
Highlight: true,
Check: func(t *testing.T, blocks []Block) {
require.Len(t, blocks, 1)
assert.Contains(t, blocks[0].Text, "<mark>zebra</mark>")
// the fragment size counts text only; the mark tags come on top.
assert.Less(t, len(blocks[0].Text), _fragmentSize+len("<mark></mark>")+10)
assert.Equal(t, main.BranchID.String()+"-p10", blocks[0].ID)
},
},
"Highlight keeps the full text of a synonym-only match": {
OrganizationID: "org-1",
Query: "accessibility",
Limit: 10,
Highlight: true,
Check: func(t *testing.T, blocks []Block) {
require.Len(t, blocks, 2)
for _, b := range blocks {
assert.NotEmpty(t, b.Text)
}
},
},
"Query missing one of three terms still matches": {
OrganizationID: "org-1",
Query: "deploy service friday",
Limit: 10,
Check: func(t *testing.T, blocks []Block) {
require.Len(t, blocks, 2)
assert.Equal(t, "we deploy the service to production every friday", blocks[0].Text)
assert.Equal(t, "production traffic hits the service after we deploy on monday", blocks[1].Text)
},
},
"Common word among unknown ones matches nothing": {
OrganizationID: "org-1",
Query: "zz-no-such-release-anywhere",
Limit: 10,
Check: func(t *testing.T, blocks []Block) {
assert.Empty(t, blocks)
},
},
"Query without a single token still matches nothing": {
OrganizationID: "org-1",
Query: "...",
Limit: 10,
Check: func(t *testing.T, blocks []Block) {
assert.Empty(t, blocks)
},
},
}
for cn, c := range cc {
t.Run(cn, func(t *testing.T) {
t.Parallel()
blocks, err := idx.search(context.Background(), c.OrganizationID, c.Query, c.Limit, c.Highlight)
require.NoError(t, err)
c.Check(t, blocks)
})
}
}
// blockTexts returns the text of each block, in order.
func blockTexts(blocks []Block) []string {
res := make([]string, 0, len(blocks))
for _, b := range blocks {
res = append(res, b.Text)
}
return res
}
func Test_Index_query(t *testing.T) {
t.Parallel()
idx := openIndex(t, indexPath(t), &stubSource{})
defer func() { require.NoError(t, idx.Close()) }()
// a query with terms: the organization filter and the text disjunction
// are required, the boosts optional.
bq, ok := idx.query("org-1", "release notes").(*query.BooleanQuery)
require.True(t, ok)
must, ok := bq.Must.(*query.ConjunctionQuery)
require.True(t, ok)
require.Len(t, must.Conjuncts, 2)
org, ok := must.Conjuncts[0].(*query.TermQuery)
require.True(t, ok)
assert.Equal(t, "org-1", org.Term)
assert.Equal(t, _fieldOrganizationID, org.FieldVal)
text, ok := must.Conjuncts[1].(*query.DisjunctionQuery)
require.True(t, ok)
require.Len(t, text.Disjuncts, 2)
phrase, ok := text.Disjuncts[0].(*query.PhraseQuery)
require.True(t, ok)
assert.Equal(t, []string{"releas", "note"}, phrase.Terms)
terms, ok := text.Disjuncts[1].(*query.DisjunctionQuery)
require.True(t, ok)
assert.Len(t, terms.Disjuncts, 2)
should, ok := bq.Should.(*query.DisjunctionQuery)
require.True(t, ok)
assert.Len(t, should.Disjuncts, 3)
// a query the analyzer reduces to nothing matches nothing.
bq, ok = idx.query("org-1", "...").(*query.BooleanQuery)
require.True(t, ok)
must, ok = bq.Must.(*query.ConjunctionQuery)
require.True(t, ok)
_, ok = must.Conjuncts[1].(*query.MatchNoneQuery)
assert.True(t, ok)
}
func Test_Index_terms(t *testing.T) {
t.Parallel()
idx := openIndex(t, indexPath(t), &stubSource{})
defer func() { require.NoError(t, idx.Close()) }()
// the terms are the analyzer's: lowercased and stemmed.
assert.Equal(t, []string{"deploy", "releas", "note"}, idx.terms("Deploying Release Notes"))
assert.Empty(t, idx.terms("..."))
// a query beyond the cap keeps its first terms only.
long := strings.TrimSpace(strings.Repeat("word ", _maxQueryTerms+5))
assert.Len(t, idx.terms(long), _maxQueryTerms)
}
func Test_termsQuery(t *testing.T) {
t.Parallel()
// one term: fuzzy or prefix, and it has to match.
dq, ok := termsQuery([]string{"kuber"}).(*query.DisjunctionQuery)
require.True(t, ok)
require.Len(t, dq.Disjuncts, 1)
assert.InDelta(t, 1, dq.Min, 0)
last, ok := dq.Disjuncts[0].(*query.DisjunctionQuery)
require.True(t, ok)
require.Len(t, last.Disjuncts, 2)
fuzzy, ok := last.Disjuncts[0].(*query.FuzzyQuery)
require.True(t, ok)
assert.Equal(t, "kuber", fuzzy.Term)
prefix, ok := last.Disjuncts[1].(*query.PrefixQuery)
require.True(t, ok)
assert.Equal(t, "kuber", prefix.Prefix)
// three terms: the first two fuzzy only, the last also a prefix, and
// all but one have to match.
dq, ok = termsQuery([]string{"deploy", "releas", "note"}).(*query.DisjunctionQuery)
require.True(t, ok)
require.Len(t, dq.Disjuncts, 3)
assert.InDelta(t, 2, dq.Min, 0)
fuzzy, ok = dq.Disjuncts[0].(*query.FuzzyQuery)
require.True(t, ok)
assert.Equal(t, "deploy", fuzzy.Term)
last, ok = dq.Disjuncts[2].(*query.DisjunctionQuery)
require.True(t, ok)
prefix, ok = last.Disjuncts[1].(*query.PrefixQuery)
require.True(t, ok)
assert.Equal(t, "note", prefix.Prefix)
}
func Test_Index_decodeHit(t *testing.T) {
t.Parallel()
documentID, branchID := xid.New(), xid.New()
fields := func(documentID, branchID string) map[string]any {
return map[string]any{
_fieldOrganizationID: "org-1",
_fieldDocumentID: documentID,
_fieldBranchID: branchID,
_fieldBranchName: "draft",
_fieldBranchDefault: false,
_fieldType: "paragraph",
_fieldText: "hello",
}
}
block := Block{
ID: branchID.String() + "-p1",
OrganizationID: "org-1",
DocumentID: documentID,
BranchID: branchID,
BranchName: "draft",
BranchDefault: false,
Type: "paragraph",
Text: "hello",
}
type tcase struct {
Fields map[string]any
Result Block
Logged string
Err error
}
cc := map[string]tcase{
"Invalid document id": {
Fields: fields("nope", branchID.String()),
Err: assert.AnError,
},
"Invalid branch id": {
Fields: fields(documentID.String(), "nope"),
Err: assert.AnError,
},
"Field of another type is logged and left empty": func() tcase {
ff := fields(documentID.String(), branchID.String())
ff[_fieldText] = 42
res := block
res.Text = ""
return tcase{Fields: ff, Result: res, Logged: "search entry field is not a string"}
}(),
"Missing boolean is logged and left false": func() tcase {
ff := fields(documentID.String(), branchID.String())
delete(ff, _fieldBranchDefault)
return tcase{Fields: ff, Result: block, Logged: "search entry field is not a boolean"}
}(),
"Successful decoding": {
Fields: fields(documentID.String(), branchID.String()),
Result: block,
},
}
for cn, c := range cc {
t.Run(cn, func(t *testing.T) {
t.Parallel()
var logs bytes.Buffer
idx := &Index{log: slog.New(slog.NewTextHandler(&logs, nil))}
res, err := idx.decodeHit(branchID.String()+"-p1", c.Fields)
if c.Err != nil {
require.Error(t, err)
return
}
require.NoError(t, err)
assert.Equal(t, c.Result, res)
if c.Logged == "" {
assert.Empty(t, logs.String())
return
}
assert.Contains(t, logs.String(), c.Logged)
})
}
}

60
server/core/internal/server/0mock_db_test.go
View File

@ -217,8 +217,8 @@ var _ DB = &DBMock{}
// InsertDocumentHookFunc: func(ctx context.Context, hk hook.Hook) error {
// panic("mock out the InsertDocumentHook method")
// },
// InsertDocumentSearchJobFunc: func(ctx context.Context, diff search.BlocksDifference) error {
// panic("mock out the InsertDocumentSearchJob method")
// InsertSearchJobFunc: func(ctx context.Context, job search.Job) error {
// panic("mock out the InsertSearchJob method")
// },
// InsertGithubInstallationFunc: func(ctx context.Context, installationID int64) error {
// panic("mock out the InsertGithubInstallation method")
@ -503,8 +503,8 @@ type DBMock struct {
// InsertDocumentHookFunc mocks the InsertDocumentHook method.
InsertDocumentHookFunc func(ctx context.Context, hk hook.Hook) error
// InsertDocumentSearchJobFunc mocks the InsertDocumentSearchJob method.
InsertDocumentSearchJobFunc func(ctx context.Context, diff search.BlocksDifference) error
// InsertSearchJobFunc mocks the InsertSearchJob method.
InsertSearchJobFunc func(ctx context.Context, job search.Job) error
// InsertGithubInstallationFunc mocks the InsertGithubInstallation method.
InsertGithubInstallationFunc func(ctx context.Context, installationID int64) error
@ -1132,12 +1132,12 @@ type DBMock struct {
// Hk is the hk argument value.
Hk hook.Hook
}
// InsertDocumentSearchJob holds details about calls to the InsertDocumentSearchJob method.
InsertDocumentSearchJob []struct {
// InsertSearchJob holds details about calls to the InsertSearchJob method.
InsertSearchJob []struct {
// Ctx is the ctx argument value.
Ctx context.Context
// Diff is the diff argument value.
Diff search.BlocksDifference
// Job is the job argument value.
Job search.Job
}
// InsertGithubInstallation holds details about calls to the InsertGithubInstallation method.
InsertGithubInstallation []struct {
@ -1458,7 +1458,7 @@ type DBMock struct {
lockInsertDocumentCommentReply sync.RWMutex
lockInsertDocumentFile sync.RWMutex
lockInsertDocumentHook sync.RWMutex
lockInsertDocumentSearchJob sync.RWMutex
lockInsertSearchJob sync.RWMutex
lockInsertGithubInstallation sync.RWMutex
lockInsertSlackApp sync.RWMutex
lockInsertSlackMessage sync.RWMutex
@ -4140,42 +4140,42 @@ func (mock *DBMock) InsertDocumentHookCalls() []struct {
return calls
}
// InsertDocumentSearchJob calls InsertDocumentSearchJobFunc.
func (mock *DBMock) InsertDocumentSearchJob(ctx context.Context, diff search.BlocksDifference) error {
// InsertSearchJob calls InsertSearchJobFunc.
func (mock *DBMock) InsertSearchJob(ctx context.Context, job search.Job) error {
callInfo := struct {
Ctx context.Context
Diff search.BlocksDifference
Ctx context.Context
Job search.Job
}{
Ctx: ctx,
Diff: diff,
Ctx: ctx,
Job: job,
}
mock.lockInsertDocumentSearchJob.Lock()
mock.calls.InsertDocumentSearchJob = append(mock.calls.InsertDocumentSearchJob, callInfo)
mock.lockInsertDocumentSearchJob.Unlock()
if mock.InsertDocumentSearchJobFunc == nil {
mock.lockInsertSearchJob.Lock()
mock.calls.InsertSearchJob = append(mock.calls.InsertSearchJob, callInfo)
mock.lockInsertSearchJob.Unlock()
if mock.InsertSearchJobFunc == nil {
var (
errOut error
)
return errOut
}
return mock.InsertDocumentSearchJobFunc(ctx, diff)
return mock.InsertSearchJobFunc(ctx, job)
}
// InsertDocumentSearchJobCalls gets all the calls that were made to InsertDocumentSearchJob.
// InsertSearchJobCalls gets all the calls that were made to InsertSearchJob.
// Check the length with:
//
// len(mockedDB.InsertDocumentSearchJobCalls())
func (mock *DBMock) InsertDocumentSearchJobCalls() []struct {
Ctx context.Context
Diff search.BlocksDifference
// len(mockedDB.InsertSearchJobCalls())
func (mock *DBMock) InsertSearchJobCalls() []struct {
Ctx context.Context
Job search.Job
} {
var calls []struct {
Ctx context.Context
Diff search.BlocksDifference
Ctx context.Context
Job search.Job
}
mock.lockInsertDocumentSearchJob.RLock()
calls = mock.calls.InsertDocumentSearchJob
mock.lockInsertDocumentSearchJob.RUnlock()
mock.lockInsertSearchJob.RLock()
calls = mock.calls.InsertSearchJob
mock.lockInsertSearchJob.RUnlock()
return calls
}

67
server/core/internal/server/0mock_search_trigger_test.go Normal file
View File

@ -0,0 +1,67 @@
// Code generated by moq; DO NOT EDIT.
// github.com/matryer/moq
package server
import (
"sync"
)
// Ensure, that SearchTriggerMock does implement SearchTrigger.
// If this is not the case, regenerate this file with moq.
var _ SearchTrigger = &SearchTriggerMock{}
// SearchTriggerMock is a mock implementation of SearchTrigger.
//
// func TestSomethingThatUsesSearchTrigger(t *testing.T) {
//
// // make and configure a mocked SearchTrigger
// mockedSearchTrigger := &SearchTriggerMock{
// TriggerFunc: func() {
// panic("mock out the Trigger method")
// },
// }
//
// // use mockedSearchTrigger in code that requires SearchTrigger
// // and then make assertions.
//
// }
type SearchTriggerMock struct {
// TriggerFunc mocks the Trigger method.
TriggerFunc func()
// calls tracks calls to the methods.
calls struct {
// Trigger holds details about calls to the Trigger method.
Trigger []struct {
}
}
lockTrigger sync.RWMutex
}
// Trigger calls TriggerFunc.
func (mock *SearchTriggerMock) Trigger() {
callInfo := struct {
}{}
mock.lockTrigger.Lock()
mock.calls.Trigger = append(mock.calls.Trigger, callInfo)
mock.lockTrigger.Unlock()
if mock.TriggerFunc == nil {
return
}
mock.TriggerFunc()
}
// TriggerCalls gets all the calls that were made to Trigger.
// Check the length with:
//
// len(mockedSearchTrigger.TriggerCalls())
func (mock *SearchTriggerMock) TriggerCalls() []struct {
} {
var calls []struct {
}
mock.lockTrigger.RLock()
calls = mock.calls.Trigger
mock.lockTrigger.RUnlock()
return calls
}

3
server/core/internal/server/capabilities.go
View File

@ -25,9 +25,6 @@ type Capabilities struct {
// ChangeDetection reports whether the changedetection.io integration
// is configured.
ChangeDetection bool `json:"changeDetection"`
// Search reports whether document search is configured.
Search bool `json:"search"`
}
// AssistantCapability describes the AI assistant's availability,

6
server/core/internal/server/capabilities_test.go
View File

@ -15,8 +15,7 @@ func Test_Server_fetchCapabilities(t *testing.T) {
s := Server{
log: discardLog(),
capabilities: Capabilities{
Slack: true,
Search: true,
Slack: true,
AIAssistant: AssistantCapability{
Status: provider.StatusActiveButWeak,
Model: "claude-sonnet-5",
@ -36,8 +35,7 @@ func Test_Server_fetchCapabilities(t *testing.T) {
"github": false,
"slack": true,
"aiAssistant": {"status": "active-but-weak", "model": "claude-sonnet-5"},
"changeDetection": false,
"search": true
"changeDetection": false
}`,
rec.Body.String(),
)

2
server/core/internal/server/internal/assistant/http_test.go
View File

@ -22,7 +22,7 @@ import (
)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
goleak.VerifyTestMain(m, testutil.IgnoreBleveWorkers())
}
// chatServer serves HandleChat with a test session on every request and

60
server/core/internal/server/internal/document/0mock_db_test.go
View File

@ -109,8 +109,8 @@ var _ DB = &DBMock{}
// InsertDocumentHookFunc: func(ctx context.Context, hk hook.Hook) error {
// panic("mock out the InsertDocumentHook method")
// },
// InsertDocumentSearchJobFunc: func(ctx context.Context, diff search.BlocksDifference) error {
// panic("mock out the InsertDocumentSearchJob method")
// InsertSearchJobFunc: func(ctx context.Context, job search.Job) error {
// panic("mock out the InsertSearchJob method")
// },
// PromoteBranchApprovalsFunc: func(ctx context.Context, fromBranchID xid.ID, toBranchID xid.ID, organizationID string) error {
// panic("mock out the PromoteBranchApprovals method")
@ -230,8 +230,8 @@ type DBMock struct {
// InsertDocumentHookFunc mocks the InsertDocumentHook method.
InsertDocumentHookFunc func(ctx context.Context, hk hook.Hook) error
// InsertDocumentSearchJobFunc mocks the InsertDocumentSearchJob method.
InsertDocumentSearchJobFunc func(ctx context.Context, diff search.BlocksDifference) error
// InsertSearchJobFunc mocks the InsertSearchJob method.
InsertSearchJobFunc func(ctx context.Context, job search.Job) error
// PromoteBranchApprovalsFunc mocks the PromoteBranchApprovals method.
PromoteBranchApprovalsFunc func(ctx context.Context, fromBranchID xid.ID, toBranchID xid.ID, organizationID string) error
@ -506,12 +506,12 @@ type DBMock struct {
// Hk is the hk argument value.
Hk hook.Hook
}
// InsertDocumentSearchJob holds details about calls to the InsertDocumentSearchJob method.
InsertDocumentSearchJob []struct {
// InsertSearchJob holds details about calls to the InsertSearchJob method.
InsertSearchJob []struct {
// Ctx is the ctx argument value.
Ctx context.Context
// Diff is the diff argument value.
Diff search.BlocksDifference
// Job is the job argument value.
Job search.Job
}
// PromoteBranchApprovals holds details about calls to the PromoteBranchApprovals method.
PromoteBranchApprovals []struct {
@ -625,7 +625,7 @@ type DBMock struct {
lockInsertDocumentBranch sync.RWMutex
lockInsertDocumentFile sync.RWMutex
lockInsertDocumentHook sync.RWMutex
lockInsertDocumentSearchJob sync.RWMutex
lockInsertSearchJob sync.RWMutex
lockPromoteBranchApprovals sync.RWMutex
lockReplaceBranchTags sync.RWMutex
lockDetachDocumentHooksByBranchID sync.RWMutex
@ -1840,42 +1840,42 @@ func (mock *DBMock) InsertDocumentHookCalls() []struct {
return calls
}
// InsertDocumentSearchJob calls InsertDocumentSearchJobFunc.
func (mock *DBMock) InsertDocumentSearchJob(ctx context.Context, diff search.BlocksDifference) error {
// InsertSearchJob calls InsertSearchJobFunc.
func (mock *DBMock) InsertSearchJob(ctx context.Context, job search.Job) error {
callInfo := struct {
Ctx context.Context
Diff search.BlocksDifference
Ctx context.Context
Job search.Job
}{
Ctx: ctx,
Diff: diff,
Ctx: ctx,
Job: job,
}
mock.lockInsertDocumentSearchJob.Lock()
mock.calls.InsertDocumentSearchJob = append(mock.calls.InsertDocumentSearchJob, callInfo)
mock.lockInsertDocumentSearchJob.Unlock()
if mock.InsertDocumentSearchJobFunc == nil {
mock.lockInsertSearchJob.Lock()
mock.calls.InsertSearchJob = append(mock.calls.InsertSearchJob, callInfo)
mock.lockInsertSearchJob.Unlock()
if mock.InsertSearchJobFunc == nil {
var (
errOut error
)
return errOut
}
return mock.InsertDocumentSearchJobFunc(ctx, diff)
return mock.InsertSearchJobFunc(ctx, job)
}
// InsertDocumentSearchJobCalls gets all the calls that were made to InsertDocumentSearchJob.
// InsertSearchJobCalls gets all the calls that were made to InsertSearchJob.
// Check the length with:
//
// len(mockedDB.InsertDocumentSearchJobCalls())
func (mock *DBMock) InsertDocumentSearchJobCalls() []struct {
Ctx context.Context
Diff search.BlocksDifference
// len(mockedDB.InsertSearchJobCalls())
func (mock *DBMock) InsertSearchJobCalls() []struct {
Ctx context.Context
Job search.Job
} {
var calls []struct {
Ctx context.Context
Diff search.BlocksDifference
Ctx context.Context
Job search.Job
}
mock.lockInsertDocumentSearchJob.RLock()
calls = mock.calls.InsertDocumentSearchJob
mock.lockInsertDocumentSearchJob.RUnlock()
mock.lockInsertSearchJob.RLock()
calls = mock.calls.InsertSearchJob
mock.lockInsertSearchJob.RUnlock()
return calls
}

67
server/core/internal/server/internal/document/0mock_search_trigger_test.go Normal file
View File

@ -0,0 +1,67 @@
// Code generated by moq; DO NOT EDIT.
// github.com/matryer/moq
package document
import (
"sync"
)
// Ensure, that SearchTriggerMock does implement SearchTrigger.
// If this is not the case, regenerate this file with moq.
var _ SearchTrigger = &SearchTriggerMock{}
// SearchTriggerMock is a mock implementation of SearchTrigger.
//
// func TestSomethingThatUsesSearchTrigger(t *testing.T) {
//
// // make and configure a mocked SearchTrigger
// mockedSearchTrigger := &SearchTriggerMock{
// TriggerFunc: func() {
// panic("mock out the Trigger method")
// },
// }
//
// // use mockedSearchTrigger in code that requires SearchTrigger
// // and then make assertions.
//
// }
type SearchTriggerMock struct {
// TriggerFunc mocks the Trigger method.
TriggerFunc func()
// calls tracks calls to the methods.
calls struct {
// Trigger holds details about calls to the Trigger method.
Trigger []struct {
}
}
lockTrigger sync.RWMutex
}
// Trigger calls TriggerFunc.
func (mock *SearchTriggerMock) Trigger() {
callInfo := struct {
}{}
mock.lockTrigger.Lock()
mock.calls.Trigger = append(mock.calls.Trigger, callInfo)
mock.lockTrigger.Unlock()
if mock.TriggerFunc == nil {
return
}
mock.TriggerFunc()
}
// TriggerCalls gets all the calls that were made to Trigger.
// Check the length with:
//
// len(mockedSearchTrigger.TriggerCalls())
func (mock *SearchTriggerMock) TriggerCalls() []struct {
} {
var calls []struct {
}
mock.lockTrigger.RLock()
calls = mock.calls.Trigger
mock.lockTrigger.RUnlock()
return calls
}

65
server/core/internal/server/internal/document/0mock_search_gateway_test.go → server/core/internal/server/internal/document/0mock_searcher_test.go
View File

@ -8,37 +8,31 @@ import (
"sync"
)
// Ensure, that SearchGatewayMock does implement SearchGateway.
// Ensure, that SearcherMock does implement Searcher.
// If this is not the case, regenerate this file with moq.
var _ SearchGateway = &SearchGatewayMock{}
var _ Searcher = &SearcherMock{}
// SearchGatewayMock is a mock implementation of SearchGateway.
// SearcherMock is a mock implementation of Searcher.
//
// func TestSomethingThatUsesSearchGateway(t *testing.T) {
// func TestSomethingThatUsesSearcher(t *testing.T) {
//
// // make and configure a mocked SearchGateway
// mockedSearchGateway := &SearchGatewayMock{
// // make and configure a mocked Searcher
// mockedSearcher := &SearcherMock{
// SearchDocumentsFunc: func(ctx context.Context, organizationID string, query string) ([]byte, error) {
// panic("mock out the SearchDocuments method")
// },
// }
//
// // use mockedSearchGateway in code that requires SearchGateway
// // use mockedSearcher in code that requires Searcher
// // and then make assertions.
//
// }
type SearchGatewayMock struct {
// ConfiguredFunc mocks the Configured method.
ConfiguredFunc func() bool
type SearcherMock struct {
// SearchDocumentsFunc mocks the SearchDocuments method.
SearchDocumentsFunc func(ctx context.Context, organizationID string, query string) ([]byte, error)
// calls tracks calls to the methods.
calls struct {
// Configured holds details about calls to the Configured method.
Configured []struct {
}
// SearchDocuments holds details about calls to the SearchDocuments method.
SearchDocuments []struct {
// Ctx is the ctx argument value.
@ -49,12 +43,11 @@ type SearchGatewayMock struct {
Query string
}
}
lockConfigured sync.RWMutex
lockSearchDocuments sync.RWMutex
}
// SearchDocuments calls SearchDocumentsFunc.
func (mock *SearchGatewayMock) SearchDocuments(ctx context.Context, organizationID string, query string) ([]byte, error) {
func (mock *SearcherMock) SearchDocuments(ctx context.Context, organizationID string, query string) ([]byte, error) {
callInfo := struct {
Ctx context.Context
OrganizationID string
@ -69,10 +62,10 @@ func (mock *SearchGatewayMock) SearchDocuments(ctx context.Context, organization
mock.lockSearchDocuments.Unlock()
if mock.SearchDocumentsFunc == nil {
var (
bytesOut []byte
errOut error
out0 []byte
errOut error
)
return bytesOut, errOut
return out0, errOut
}
return mock.SearchDocumentsFunc(ctx, organizationID, query)
}
@ -80,8 +73,8 @@ func (mock *SearchGatewayMock) SearchDocuments(ctx context.Context, organization
// SearchDocumentsCalls gets all the calls that were made to SearchDocuments.
// Check the length with:
//
// len(mockedSearchGateway.SearchDocumentsCalls())
func (mock *SearchGatewayMock) SearchDocumentsCalls() []struct {
// len(mockedSearcher.SearchDocumentsCalls())
func (mock *SearcherMock) SearchDocumentsCalls() []struct {
Ctx context.Context
OrganizationID string
Query string
@ -96,33 +89,3 @@ func (mock *SearchGatewayMock) SearchDocumentsCalls() []struct {
mock.lockSearchDocuments.RUnlock()
return calls
}
// Configured calls ConfiguredFunc.
func (mock *SearchGatewayMock) Configured() bool {
callInfo := struct {
}{}
mock.lockConfigured.Lock()
mock.calls.Configured = append(mock.calls.Configured, callInfo)
mock.lockConfigured.Unlock()
if mock.ConfiguredFunc == nil {
var (
bOut bool
)
return bOut
}
return mock.ConfiguredFunc()
}
// ConfiguredCalls gets all the calls that were made to Configured.
// Check the length with:
//
// len(mockedSearchGateway.ConfiguredCalls())
func (mock *SearchGatewayMock) ConfiguredCalls() []struct {
} {
var calls []struct {
}
mock.lockConfigured.RLock()
calls = mock.calls.Configured
mock.lockConfigured.RUnlock()
return calls
}

60
server/core/internal/server/internal/document/0mock_tx_test.go
View File

@ -109,8 +109,8 @@ var _ Tx = &TxMock{}
// InsertDocumentHookFunc: func(ctx context.Context, hk hook.Hook) error {
// panic("mock out the InsertDocumentHook method")
// },
// InsertDocumentSearchJobFunc: func(ctx context.Context, diff search.BlocksDifference) error {
// panic("mock out the InsertDocumentSearchJob method")
// InsertSearchJobFunc: func(ctx context.Context, job search.Job) error {
// panic("mock out the InsertSearchJob method")
// },
// PromoteBranchApprovalsFunc: func(ctx context.Context, fromBranchID xid.ID, toBranchID xid.ID, organizationID string) error {
// panic("mock out the PromoteBranchApprovals method")
@ -233,8 +233,8 @@ type TxMock struct {
// InsertDocumentHookFunc mocks the InsertDocumentHook method.
InsertDocumentHookFunc func(ctx context.Context, hk hook.Hook) error
// InsertDocumentSearchJobFunc mocks the InsertDocumentSearchJob method.
InsertDocumentSearchJobFunc func(ctx context.Context, diff search.BlocksDifference) error
// InsertSearchJobFunc mocks the InsertSearchJob method.
InsertSearchJobFunc func(ctx context.Context, job search.Job) error
// PromoteBranchApprovalsFunc mocks the PromoteBranchApprovals method.
PromoteBranchApprovalsFunc func(ctx context.Context, fromBranchID xid.ID, toBranchID xid.ID, organizationID string) error
@ -508,12 +508,12 @@ type TxMock struct {
// Hk is the hk argument value.
Hk hook.Hook
}
// InsertDocumentSearchJob holds details about calls to the InsertDocumentSearchJob method.
InsertDocumentSearchJob []struct {
// InsertSearchJob holds details about calls to the InsertSearchJob method.
InsertSearchJob []struct {
// Ctx is the ctx argument value.
Ctx context.Context
// Diff is the diff argument value.
Diff search.BlocksDifference
// Job is the job argument value.
Job search.Job
}
// PromoteBranchApprovals holds details about calls to the PromoteBranchApprovals method.
PromoteBranchApprovals []struct {
@ -630,7 +630,7 @@ type TxMock struct {
lockInsertDocumentBranch sync.RWMutex
lockInsertDocumentFile sync.RWMutex
lockInsertDocumentHook sync.RWMutex
lockInsertDocumentSearchJob sync.RWMutex
lockInsertSearchJob sync.RWMutex
lockPromoteBranchApprovals sync.RWMutex
lockReplaceBranchTags sync.RWMutex
lockRollback sync.RWMutex
@ -1837,42 +1837,42 @@ func (mock *TxMock) InsertDocumentHookCalls() []struct {
return calls
}
// InsertDocumentSearchJob calls InsertDocumentSearchJobFunc.
func (mock *TxMock) InsertDocumentSearchJob(ctx context.Context, diff search.BlocksDifference) error {
// InsertSearchJob calls InsertSearchJobFunc.
func (mock *TxMock) InsertSearchJob(ctx context.Context, job search.Job) error {
callInfo := struct {
Ctx context.Context
Diff search.BlocksDifference
Ctx context.Context
Job search.Job
}{
Ctx: ctx,
Diff: diff,
Ctx: ctx,
Job: job,
}
mock.lockInsertDocumentSearchJob.Lock()
mock.calls.InsertDocumentSearchJob = append(mock.calls.InsertDocumentSearchJob, callInfo)
mock.lockInsertDocumentSearchJob.Unlock()
if mock.InsertDocumentSearchJobFunc == nil {
mock.lockInsertSearchJob.Lock()
mock.calls.InsertSearchJob = append(mock.calls.InsertSearchJob, callInfo)
mock.lockInsertSearchJob.Unlock()
if mock.InsertSearchJobFunc == nil {
var (
errOut error
)
return errOut
}
return mock.InsertDocumentSearchJobFunc(ctx, diff)
return mock.InsertSearchJobFunc(ctx, job)
}
// InsertDocumentSearchJobCalls gets all the calls that were made to InsertDocumentSearchJob.
// InsertSearchJobCalls gets all the calls that were made to InsertSearchJob.
// Check the length with:
//
// len(mockedTx.InsertDocumentSearchJobCalls())
func (mock *TxMock) InsertDocumentSearchJobCalls() []struct {
Ctx context.Context
Diff search.BlocksDifference
// len(mockedTx.InsertSearchJobCalls())
func (mock *TxMock) InsertSearchJobCalls() []struct {
Ctx context.Context
Job search.Job
} {
var calls []struct {
Ctx context.Context
Diff search.BlocksDifference
Ctx context.Context
Job search.Job
}
mock.lockInsertDocumentSearchJob.RLock()
calls = mock.calls.InsertDocumentSearchJob
mock.lockInsertDocumentSearchJob.RUnlock()
mock.lockInsertSearchJob.RLock()
calls = mock.calls.InsertSearchJob
mock.lockInsertSearchJob.RUnlock()
return calls
}

26
server/core/internal/server/internal/document/branch_http.go
View File

@ -84,7 +84,7 @@ func (h *Handler) UpdateDocumentBranch(w http.ResponseWriter, r *http.Request) {
// the index carries the branch name on every entry of the branch.
if ndoc.BranchName != doc.BranchName {
if err := h.searchJobs.Enqueue(r.Context(), tx, search.BlocksDiff(doc.Search(), ndoc.Search())); err != nil {
if err := tx.InsertSearchJob(r.Context(), search.BranchScope(session.ActiveOrganizationID, ndoc.ID, ndoc.BranchID)); err != nil {
httpserver.RespondError(h.log, w, err)
return
}
@ -95,6 +95,8 @@ func (h *Handler) UpdateDocumentBranch(w http.ResponseWriter, r *http.Request) {
return
}
h.searchTrigger.Trigger()
if h.metadata.changeCallback != nil {
h.metadata.changeCallback(session.ActiveOrganizationID, ndoc)
}
@ -228,11 +230,7 @@ func (h *Handler) UpdateDocumentBranchByIDUnsafe(w http.ResponseWriter, r *http.
}
}
if err = h.searchJobs.Enqueue(
r.Context(),
tx,
search.BlocksDiff(doc.Search(), ndoc.Search()),
); err != nil {
if err = tx.InsertSearchJob(r.Context(), search.BranchScope(doc.OrganizationID, doc.ID, doc.BranchID)); err != nil {
httpserver.RespondError(h.log, w, err)
return
}
@ -243,6 +241,8 @@ func (h *Handler) UpdateDocumentBranchByIDUnsafe(w http.ResponseWriter, r *http.
return
}
h.searchTrigger.Trigger()
if h.metadata.changeCallback != nil {
h.metadata.changeCallback(doc.OrganizationID, ndoc)
}
@ -341,7 +341,7 @@ func (h *Handler) MergeBranches(w http.ResponseWriter, r *http.Request) {
return
}
if err := h.searchJobs.Enqueue(r.Context(), tx, search.BlocksDiff(toDoc.Search(), ndoc.Search())); err != nil {
if err := tx.InsertSearchJob(r.Context(), search.BranchScope(session.ActiveOrganizationID, toDoc.ID, toDoc.BranchID)); err != nil {
httpserver.RespondError(h.log, w, err)
return
}
@ -356,6 +356,8 @@ func (h *Handler) MergeBranches(w http.ResponseWriter, r *http.Request) {
return
}
h.searchTrigger.Trigger()
h.copyHooksToBranch(
r.Context(),
fromDoc.BranchID,
@ -469,7 +471,7 @@ func (h *Handler) CreateDocumentBranch(w http.ResponseWriter, r *http.Request) {
return
}
if err := h.searchJobs.Enqueue(r.Context(), tx, search.BlocksDiff(nil, newDoc.Search())); err != nil {
if err := tx.InsertSearchJob(r.Context(), search.BranchScope(session.ActiveOrganizationID, newDoc.ID, newDoc.BranchID)); err != nil {
httpserver.RespondError(h.log, w, err)
return
}
@ -479,6 +481,8 @@ func (h *Handler) CreateDocumentBranch(w http.ResponseWriter, r *http.Request) {
return
}
h.searchTrigger.Trigger()
h.copyHooksToBranch(
r.Context(),
sourceDoc.BranchID,
@ -557,9 +561,7 @@ func (h *Handler) DeleteDocumentBranch(w http.ResponseWriter, r *http.Request) {
return
}
if err := h.searchJobs.Enqueue(r.Context(), tx, search.BlocksDifference{
RemovedBranches: []search.BranchRemoval{{DocumentID: branchDoc.ID, BranchID: branchID}},
}); err != nil {
if err := tx.InsertSearchJob(r.Context(), search.BranchScope(session.ActiveOrganizationID, branchDoc.ID, branchID)); err != nil {
httpserver.RespondError(h.log, w, err)
return
}
@ -569,6 +571,8 @@ func (h *Handler) DeleteDocumentBranch(w http.ResponseWriter, r *http.Request) {
return
}
h.searchTrigger.Trigger()
httpserver.Respond(
h.log,
w,

60
server/core/internal/server/internal/document/branch_http_test.go
View File

@ -5,7 +5,6 @@ import (
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/guregu/null/v5"
@ -227,7 +226,7 @@ func Test_Handler_UpdateDocumentBranchByIDUnsafe(t *testing.T) {
FetchDocumentMaintainersFunc: func(context.Context, xid.ID, string) ([]string, error) {
return []string{"u1", "u9"}, nil
},
InsertDocumentSearchJobFunc: func(context.Context, search.BlocksDifference) error {
InsertSearchJobFunc: func(context.Context, search.Job) error {
return errors.New("boom")
},
},
@ -307,18 +306,14 @@ func Test_Handler_UpdateDocumentBranchByIDUnsafe(t *testing.T) {
assert.Len(t, c.Tx.CommitCalls(), c.Committed)
assert.Equal(t, c.Metadata, cnt.metadata)
assert.Equal(t, c.Maintainers, cnt.maintainers)
assert.Len(t, c.Tx.InsertDocumentSearchJobCalls(), c.SearchJobs)
assert.Len(t, c.Tx.InsertSearchJobCalls(), c.SearchJobs)
// the diff is scoped to the persisted branch: entries are keyed
// by its id, whichever branch it is.
// the job is scoped to the persisted branch, whichever branch
// it is.
if !c.BranchID.IsZero() {
diff := c.Tx.InsertDocumentSearchJobCalls()[0].Diff
require.NotEmpty(t, diff.Updated)
for _, b := range diff.Updated {
assert.Equal(t, c.BranchID, b.BranchID)
assert.True(t, strings.HasPrefix(b.ID, c.BranchID.String()+"-"))
}
job := c.Tx.InsertSearchJobCalls()[0].Job
assert.Equal(t, c.BranchID, job.BranchID.V)
assert.True(t, job.DocumentID.Valid)
}
})
}
@ -782,7 +777,7 @@ func Test_Handler_CreateDocumentBranch(t *testing.T) {
},
},
Tx: &TxMock{
InsertDocumentSearchJobFunc: func(context.Context, search.BlocksDifference) error {
InsertSearchJobFunc: func(context.Context, search.Job) error {
return errors.New("boom")
},
},
@ -846,16 +841,10 @@ func Test_Handler_CreateDocumentBranch(t *testing.T) {
assert.Equal(t, _branchID, c.Tx.CopyBranchTagsCalls()[0].FromBranchID)
assert.Equal(t, forked.BranchID, c.Tx.CopyBranchTagsCalls()[0].ToBranchID)
// the fork's entries are added under its own branch id before
// the commit, so the branch is searchable without an edit.
require.Len(t, c.Tx.InsertDocumentSearchJobCalls(), 1)
diff := c.Tx.InsertDocumentSearchJobCalls()[0].Diff
require.NotEmpty(t, diff.Added)
for _, b := range diff.Added {
assert.Equal(t, forked.BranchID, b.BranchID)
assert.True(t, strings.HasPrefix(b.ID, forked.BranchID.String()+"-"))
}
// the fork is queued under its own branch id before the
// commit, so the branch is searchable without an edit.
require.Len(t, c.Tx.InsertSearchJobCalls(), 1)
assert.Equal(t, search.BranchScope("org1", forked.ID, forked.BranchID), c.Tx.InsertSearchJobCalls()[0].Job)
}
if len(c.CopiedHooks) == 0 {
@ -978,7 +967,7 @@ func Test_Handler_DeleteDocumentBranch(t *testing.T) {
},
},
Tx: &TxMock{
InsertDocumentSearchJobFunc: func(context.Context, search.BlocksDifference) error {
InsertSearchJobFunc: func(context.Context, search.Job) error {
return errors.New("boom")
},
},
@ -1024,13 +1013,11 @@ func Test_Handler_DeleteDocumentBranch(t *testing.T) {
assert.Len(t, tx.DeleteDocumentBranchByIDCalls(), c.Deleted)
assert.Len(t, tx.CommitCalls(), c.Committed)
// the branch row is gone with its content, so the index is
// cleared by branch id.
// the branch row is gone with its content, so the worker finds
// nothing under the scope and clears it.
if c.RespCode == http.StatusNoContent {
require.Len(t, tx.InsertDocumentSearchJobCalls(), 1)
assert.Equal(t, search.BlocksDifference{
RemovedBranches: []search.BranchRemoval{{DocumentID: _documentID, BranchID: _branchID}},
}, tx.InsertDocumentSearchJobCalls()[0].Diff)
require.Len(t, tx.InsertSearchJobCalls(), 1)
assert.Equal(t, search.BranchScope("org1", _documentID, _branchID), tx.InsertSearchJobCalls()[0].Job)
}
})
}
@ -1203,7 +1190,7 @@ func Test_Handler_UpdateDocumentBranch(t *testing.T) {
},
},
Tx: &TxMock{
InsertDocumentSearchJobFunc: func(context.Context, search.BlocksDifference) error {
InsertSearchJobFunc: func(context.Context, search.Job) error {
return errors.New("boom")
},
},
@ -1268,7 +1255,7 @@ func Test_Handler_UpdateDocumentBranch(t *testing.T) {
assert.Equal(t, c.RespCode, rec.Code)
assert.Len(t, tx.UpdateDocumentBranchMetadataCalls(), c.Updated)
assert.Len(t, tx.InsertDocumentSearchJobCalls(), c.Jobs)
assert.Len(t, tx.InsertSearchJobCalls(), c.Jobs)
assert.Equal(t, c.Metadata, cnt.metadata)
if c.RespCode == http.StatusOK {
@ -1279,13 +1266,10 @@ func Test_Handler_UpdateDocumentBranch(t *testing.T) {
assert.Len(t, tx.CommitCalls(), 1)
}
// a rename rewrites every entry of the branch, since each one
// carries the branch name.
// a rename resyncs the branch, since every entry carries the
// branch name.
if c.Jobs == 1 && c.RespCode == http.StatusOK {
diff := tx.InsertDocumentSearchJobCalls()[0].Diff
require.Len(t, diff.Updated, 1)
assert.Equal(t, _branchID.String()+"-docname", diff.Updated[0].ID)
assert.Equal(t, "Renamed", diff.Updated[0].BranchName)
assert.Equal(t, search.BranchScope("org1", _documentID, _branchID), tx.InsertSearchJobCalls()[0].Job)
}
})
}

61
server/core/internal/server/internal/document/document_http.go
View File

@ -7,6 +7,7 @@ import (
"net/http"
"net/url"
"path"
"unicode/utf8"
"github.com/guregu/null/v5"
"github.com/oxynote/oxynote/server/core/internal/apps/github"
@ -24,6 +25,11 @@ import (
"github.com/rs/xid"
)
// _searchQueryMaxLength caps a search query in characters. Every term
// expands to a dictionary walk inside the process, so the query size
// bounds the work one request can ask for.
const _searchQueryMaxLength = 200
// ErrInvalidSearchQuery is returned when the search query is invalid.
var ErrInvalidSearchQuery = errutil.New(http.StatusBadRequest, "document.invalid_search_query", "invalid search query")
@ -44,8 +50,8 @@ type Handler struct {
db DB
githubMan *github.Manager
webchangeClient *webchange.Client
searchGateway SearchGateway
searchJobs *search.Jobs
searcher Searcher
searchTrigger SearchTrigger
notifPub notification.Publisher
storer Storer
@ -72,8 +78,8 @@ func NewHandler(
db DB,
githubMan *github.Manager,
webchangeClient *webchange.Client,
searchGateway SearchGateway,
searchJobs *search.Jobs,
searcher Searcher,
searchTrigger SearchTrigger,
notifPub notification.Publisher,
storer Storer,
) *Handler {
@ -82,8 +88,8 @@ func NewHandler(
db: db,
githubMan: githubMan,
webchangeClient: webchangeClient,
searchGateway: searchGateway,
searchJobs: searchJobs,
searcher: searcher,
searchTrigger: searchTrigger,
notifPub: notifPub,
storer: storer,
}
@ -515,7 +521,7 @@ func (h *Handler) SearchDocuments(w http.ResponseWriter, r *http.Request) {
}
q := r.URL.Query().Get("q")
if q == "" {
if q == "" || utf8.RuneCountInString(q) > _searchQueryMaxLength {
httpserver.RespondError(
h.log,
w,
@ -525,7 +531,7 @@ func (h *Handler) SearchDocuments(w http.ResponseWriter, r *http.Request) {
return
}
data, err := h.searchGateway.SearchDocuments(r.Context(), session.ActiveOrganizationID, q)
data, err := h.searcher.SearchDocuments(r.Context(), session.ActiveOrganizationID, q)
if err != nil {
httpserver.RespondError(h.log, w, err)
return
@ -691,10 +697,8 @@ func (h *Handler) DeleteDocument(w http.ResponseWriter, r *http.Request) {
return
}
if len(ids) != 0 {
if err = h.searchJobs.Enqueue(r.Context(), tx, search.BlocksDifference{
RemovedDocuments: ids,
}); err != nil {
for _, id := range ids {
if err = tx.InsertSearchJob(r.Context(), search.DocumentScope(session.ActiveOrganizationID, id)); err != nil {
httpserver.RespondError(h.log, w, err)
return
}
@ -706,6 +710,8 @@ func (h *Handler) DeleteDocument(w http.ResponseWriter, r *http.Request) {
return
}
h.searchTrigger.Trigger()
if h.tree.changeCallback != nil {
h.tree.changeCallback(session.ActiveOrganizationID, doc.ParentID)
}
@ -903,7 +909,7 @@ func (h *Handler) insertDocumentTx(
return err
}
if err := h.searchJobs.Enqueue(ctx, tx, search.BlocksDiff(nil, doc.Search())); err != nil {
if err := tx.InsertSearchJob(ctx, search.BranchScope(session.ActiveOrganizationID, doc.ID, doc.BranchID)); err != nil {
return err
}
@ -921,7 +927,13 @@ func (h *Handler) insertDocumentTx(
return err
}
return tx.Commit()
if err = tx.Commit(); err != nil {
return err
}
h.searchTrigger.Trigger()
return nil
}
// DB is an interface that combines sqlutil.DB and DBAgent.
@ -979,8 +991,8 @@ type DocumentsDBAgent interface {
// InsertDocument should insert the document.
InsertDocument(ctx context.Context, doc documentCore.Document) error
// InsertDocumentSearchJob should insert the document search job.
InsertDocumentSearchJob(ctx context.Context, diff search.BlocksDifference) error
// InsertSearchJob should queue the search job's scope.
InsertSearchJob(ctx context.Context, job search.Job) error
// CheckDocumentExists returns nil if the document exists and belongs to the given organization.
CheckDocumentExists(ctx context.Context, id xid.ID, organizationID string) error
@ -1112,14 +1124,19 @@ type Storer interface {
Copy(ctx context.Context, srcFolder, srcID, dstFolder, dstID string) error
}
// SearchGateway is an interface that handles communication with the search engine.
// SearchTrigger runs the search-job worker once a job has committed.
//
//go:generate ../../../../scripts/codegen/mock -t internal SearchGateway
type SearchGateway interface {
// Configured should report whether search is configured on this
// deployment.
Configured() bool
//go:generate ../../../../scripts/codegen/mock -t internal SearchTrigger search_trigger
type SearchTrigger interface {
// Trigger should run a search-job pass right away.
Trigger()
}
// Searcher is the full-text search surface the search endpoint answers
// from.
//
//go:generate ../../../../scripts/codegen/mock -t internal Searcher searcher
type Searcher interface {
// SearchDocuments should find the documents matching the query.
SearchDocuments(ctx context.Context, organizationID, query string) ([]byte, error)
}

53
server/core/internal/server/internal/document/document_http_test.go
View File

@ -27,7 +27,7 @@ import (
)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
goleak.VerifyTestMain(m, testutil.IgnoreBleveWorkers())
}
// fixed IDs used across test requests.
@ -70,7 +70,7 @@ func newTestHandler(db DB, pub *fakePublisher) (*Handler, *callbackCounts) {
notifPub: pub,
storer: &StorerMock{},
webchangeClient: webchange.NewClient("", ""),
searchJobs: search.NewJobs(true),
searchTrigger: &SearchTriggerMock{},
}
hdl.tree.changeCallback = func(string, null.Value[xid.ID]) { cnt.tree++ }
@ -152,17 +152,17 @@ func Test_NewHandler(t *testing.T) {
t.Parallel()
db := &DBMock{}
gw := &SearchGatewayMock{}
jobs := search.NewJobs(true)
searcher := &SearcherMock{}
trigger := &SearchTriggerMock{}
pub := &fakePublisher{}
st := &StorerMock{}
hdl := NewHandler(slog.New(slog.DiscardHandler), db, nil, nil, gw, jobs, pub, st)
hdl := NewHandler(slog.New(slog.DiscardHandler), db, nil, nil, searcher, trigger, pub, st)
require.NotNil(t, hdl)
assert.NotNil(t, hdl.log)
assert.Same(t, db, hdl.db)
assert.Same(t, gw, hdl.searchGateway)
assert.Same(t, jobs, hdl.searchJobs)
assert.Same(t, searcher, hdl.searcher)
assert.Same(t, trigger, hdl.searchTrigger)
assert.Same(t, pub, hdl.notifPub)
assert.Same(t, st, hdl.storer)
assert.Nil(t, hdl.githubMan)
@ -1168,24 +1168,29 @@ func Test_Handler_UpdateDocumentTree(t *testing.T) {
func Test_Handler_SearchDocuments(t *testing.T) {
cc := map[string]struct {
Gateway *SearchGatewayMock
Searcher *SearcherMock
NoSession bool
Query string
RespCode int
RespBody string
}{
"No session in context": {
Gateway: &SearchGatewayMock{},
Searcher: &SearcherMock{},
NoSession: true,
Query: "?q=test",
RespCode: http.StatusUnauthorized,
},
"Empty search query": {
Gateway: &SearchGatewayMock{},
Searcher: &SearcherMock{},
RespCode: http.StatusBadRequest,
},
"Search gateway error": {
Gateway: &SearchGatewayMock{
"Overlong search query": {
Searcher: &SearcherMock{},
Query: "?q=" + strings.Repeat("a", _searchQueryMaxLength+1),
RespCode: http.StatusBadRequest,
},
"Searcher error": {
Searcher: &SearcherMock{
SearchDocumentsFunc: func(context.Context, string, string) ([]byte, error) {
return nil, errors.New("boom")
},
@ -1193,18 +1198,8 @@ func Test_Handler_SearchDocuments(t *testing.T) {
Query: "?q=test",
RespCode: http.StatusInternalServerError,
},
"Search not configured": {
Gateway: &SearchGatewayMock{
SearchDocumentsFunc: func(context.Context, string, string) ([]byte, error) {
return nil, search.ErrNotConfigured
},
},
Query: "?q=test",
RespCode: http.StatusConflict,
RespBody: `{"code":"search.not_configured","message":"search is not configured"}`,
},
"Successful search": {
Gateway: &SearchGatewayMock{
Searcher: &SearcherMock{
SearchDocumentsFunc: func(context.Context, string, string) ([]byte, error) {
return []byte(`{"hits":[]}`), nil
},
@ -1220,7 +1215,7 @@ func Test_Handler_SearchDocuments(t *testing.T) {
t.Parallel()
hdl, _ := newTestHandler(&DBMock{}, &fakePublisher{})
hdl.searchGateway = c.Gateway
hdl.searcher = c.Searcher
req := httptest.NewRequest(http.MethodGet, "http://test.com/"+c.Query, http.NoBody)
@ -1315,7 +1310,7 @@ func Test_Handler_CreateDocument(t *testing.T) {
},
"Search job insertion error": {
Tx: &TxMock{
InsertDocumentSearchJobFunc: func(context.Context, search.BlocksDifference) error {
InsertSearchJobFunc: func(context.Context, search.Job) error {
return errors.New("boom")
},
},
@ -1469,7 +1464,7 @@ func Test_Handler_DeleteDocument(t *testing.T) {
DeleteDocumentFunc: func(context.Context, xid.ID, string) ([]xid.ID, error) {
return []xid.ID{_documentID}, nil
},
InsertDocumentSearchJobFunc: func(context.Context, search.BlocksDifference) error {
InsertSearchJobFunc: func(context.Context, search.Job) error {
return errors.New("boom")
},
},
@ -1521,11 +1516,11 @@ func Test_Handler_DeleteDocument(t *testing.T) {
require.Len(t, c.Tx.DeleteDocumentCalls(), 1)
assert.Equal(t, _documentID, c.Tx.DeleteDocumentCalls()[0].ID)
// the removal of the returned subtree ids rides the same
// the resync of the returned subtree ids rides the same
// transaction as the delete.
ff := c.Tx.InsertDocumentSearchJobCalls()
ff := c.Tx.InsertSearchJobCalls()
require.Len(t, ff, 1)
assert.Equal(t, []xid.ID{_documentID}, ff[0].Diff.RemovedDocuments)
assert.Equal(t, search.DocumentScope("org1", _documentID), ff[0].Job)
}
})
}

14
server/core/internal/server/internal/mcp/http_test.go
View File

@ -12,7 +12,6 @@ import (
"github.com/jarcoal/httpmock"
"github.com/oxynote/oxynote/server/core/internal/assistant/tools"
toolsMock "github.com/oxynote/oxynote/server/core/internal/assistant/tools/_mock"
"github.com/oxynote/oxynote/server/core/internal/search"
"github.com/oxynote/oxynote/server/core/pkg/testutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -20,7 +19,7 @@ import (
)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
goleak.VerifyTestMain(m, testutil.IgnoreBleveWorkers())
}
// _testSessionURL is the internal MCP session endpoint the mocked
@ -35,6 +34,11 @@ func discardLog() *slog.Logger {
return slog.New(slog.DiscardHandler)
}
// stubSearchTrigger satisfies the tools' SearchTrigger with a no-op.
type stubSearchTrigger struct{}
func (stubSearchTrigger) Trigger() {}
// stubToolsDB builds the tools-package DB mock every test tool set
// runs against.
func stubToolsDB() *toolsMock.DB {
@ -47,10 +51,8 @@ func stubSet(db *toolsMock.DB) *tools.Set {
return tools.New(tools.NewDeps(
discardLog(),
db,
&toolsMock.Searcher{
ConfiguredFunc: func() bool { return true },
},
search.NewJobs(true),
&toolsMock.Searcher{},
stubSearchTrigger{},
nil,
nil,
nil,

60
server/core/internal/server/internal/org/0mock_db_test.go
View File

@ -49,8 +49,8 @@ var _ DB = &DBMock{}
// InsertDocumentFunc: func(ctx context.Context, doc document.Document) error {
// panic("mock out the InsertDocument method")
// },
// InsertDocumentSearchJobFunc: func(ctx context.Context, diff search.BlocksDifference) error {
// panic("mock out the InsertDocumentSearchJob method")
// InsertSearchJobFunc: func(ctx context.Context, job search.Job) error {
// panic("mock out the InsertSearchJob method")
// },
// InsertTagFunc: func(ctx context.Context, t tag.Tag) error {
// panic("mock out the InsertTag method")
@ -92,8 +92,8 @@ type DBMock struct {
// InsertDocumentFunc mocks the InsertDocument method.
InsertDocumentFunc func(ctx context.Context, doc document.Document) error
// InsertDocumentSearchJobFunc mocks the InsertDocumentSearchJob method.
InsertDocumentSearchJobFunc func(ctx context.Context, diff search.BlocksDifference) error
// InsertSearchJobFunc mocks the InsertSearchJob method.
InsertSearchJobFunc func(ctx context.Context, job search.Job) error
// InsertTagFunc mocks the InsertTag method.
InsertTagFunc func(ctx context.Context, t tag.Tag) error
@ -168,12 +168,12 @@ type DBMock struct {
// Doc is the doc argument value.
Doc document.Document
}
// InsertDocumentSearchJob holds details about calls to the InsertDocumentSearchJob method.
InsertDocumentSearchJob []struct {
// InsertSearchJob holds details about calls to the InsertSearchJob method.
InsertSearchJob []struct {
// Ctx is the ctx argument value.
Ctx context.Context
// Diff is the diff argument value.
Diff search.BlocksDifference
// Job is the job argument value.
Job search.Job
}
// InsertTag holds details about calls to the InsertTag method.
InsertTag []struct {
@ -211,7 +211,7 @@ type DBMock struct {
lockFetchOrganizationMembers sync.RWMutex
lockInsertDataSource sync.RWMutex
lockInsertDocument sync.RWMutex
lockInsertDocumentSearchJob sync.RWMutex
lockInsertSearchJob sync.RWMutex
lockInsertTag sync.RWMutex
lockUpdateOrganizationLogo sync.RWMutex
lockUpsertDocumentMaintainers sync.RWMutex
@ -541,42 +541,42 @@ func (mock *DBMock) InsertDocumentCalls() []struct {
return calls
}
// InsertDocumentSearchJob calls InsertDocumentSearchJobFunc.
func (mock *DBMock) InsertDocumentSearchJob(ctx context.Context, diff search.BlocksDifference) error {
// InsertSearchJob calls InsertSearchJobFunc.
func (mock *DBMock) InsertSearchJob(ctx context.Context, job search.Job) error {
callInfo := struct {
Ctx context.Context
Diff search.BlocksDifference
Ctx context.Context
Job search.Job
}{
Ctx: ctx,
Diff: diff,
Ctx: ctx,
Job: job,
}
mock.lockInsertDocumentSearchJob.Lock()
mock.calls.InsertDocumentSearchJob = append(mock.calls.InsertDocumentSearchJob, callInfo)
mock.lockInsertDocumentSearchJob.Unlock()
if mock.InsertDocumentSearchJobFunc == nil {
mock.lockInsertSearchJob.Lock()
mock.calls.InsertSearchJob = append(mock.calls.InsertSearchJob, callInfo)
mock.lockInsertSearchJob.Unlock()
if mock.InsertSearchJobFunc == nil {
var (
errOut error
)
return errOut
}
return mock.InsertDocumentSearchJobFunc(ctx, diff)
return mock.InsertSearchJobFunc(ctx, job)
}
// InsertDocumentSearchJobCalls gets all the calls that were made to InsertDocumentSearchJob.
// InsertSearchJobCalls gets all the calls that were made to InsertSearchJob.
// Check the length with:
//
// len(mockedDB.InsertDocumentSearchJobCalls())
func (mock *DBMock) InsertDocumentSearchJobCalls() []struct {
Ctx context.Context
Diff search.BlocksDifference
// len(mockedDB.InsertSearchJobCalls())
func (mock *DBMock) InsertSearchJobCalls() []struct {
Ctx context.Context
Job search.Job
} {
var calls []struct {
Ctx context.Context
Diff search.BlocksDifference
Ctx context.Context
Job search.Job
}
mock.lockInsertDocumentSearchJob.RLock()
calls = mock.calls.InsertDocumentSearchJob
mock.lockInsertDocumentSearchJob.RUnlock()
mock.lockInsertSearchJob.RLock()
calls = mock.calls.InsertSearchJob
mock.lockInsertSearchJob.RUnlock()
return calls
}

67
server/core/internal/server/internal/org/0mock_search_trigger_test.go Normal file
View File

@ -0,0 +1,67 @@
// Code generated by moq; DO NOT EDIT.
// github.com/matryer/moq
package org
import (
"sync"
)
// Ensure, that SearchTriggerMock does implement SearchTrigger.
// If this is not the case, regenerate this file with moq.
var _ SearchTrigger = &SearchTriggerMock{}
// SearchTriggerMock is a mock implementation of SearchTrigger.
//
// func TestSomethingThatUsesSearchTrigger(t *testing.T) {
//
// // make and configure a mocked SearchTrigger
// mockedSearchTrigger := &SearchTriggerMock{
// TriggerFunc: func() {
// panic("mock out the Trigger method")
// },
// }
//
// // use mockedSearchTrigger in code that requires SearchTrigger
// // and then make assertions.
//
// }
type SearchTriggerMock struct {
// TriggerFunc mocks the Trigger method.
TriggerFunc func()
// calls tracks calls to the methods.
calls struct {
// Trigger holds details about calls to the Trigger method.
Trigger []struct {
}
}
lockTrigger sync.RWMutex
}
// Trigger calls TriggerFunc.
func (mock *SearchTriggerMock) Trigger() {
callInfo := struct {
}{}
mock.lockTrigger.Lock()
mock.calls.Trigger = append(mock.calls.Trigger, callInfo)
mock.lockTrigger.Unlock()
if mock.TriggerFunc == nil {
return
}
mock.TriggerFunc()
}
// TriggerCalls gets all the calls that were made to Trigger.
// Check the length with:
//
// len(mockedSearchTrigger.TriggerCalls())
func (mock *SearchTriggerMock) TriggerCalls() []struct {
} {
var calls []struct {
}
mock.lockTrigger.RLock()
calls = mock.calls.Trigger
mock.lockTrigger.RUnlock()
return calls
}

60
server/core/internal/server/internal/org/0mock_tx_test.go
View File

@ -49,8 +49,8 @@ var _ Tx = &TxMock{}
// InsertDocumentFunc: func(ctx context.Context, doc document.Document) error {
// panic("mock out the InsertDocument method")
// },
// InsertDocumentSearchJobFunc: func(ctx context.Context, diff search.BlocksDifference) error {
// panic("mock out the InsertDocumentSearchJob method")
// InsertSearchJobFunc: func(ctx context.Context, job search.Job) error {
// panic("mock out the InsertSearchJob method")
// },
// InsertTagFunc: func(ctx context.Context, t tag.Tag) error {
// panic("mock out the InsertTag method")
@ -95,8 +95,8 @@ type TxMock struct {
// InsertDocumentFunc mocks the InsertDocument method.
InsertDocumentFunc func(ctx context.Context, doc document.Document) error
// InsertDocumentSearchJobFunc mocks the InsertDocumentSearchJob method.
InsertDocumentSearchJobFunc func(ctx context.Context, diff search.BlocksDifference) error
// InsertSearchJobFunc mocks the InsertSearchJob method.
InsertSearchJobFunc func(ctx context.Context, job search.Job) error
// InsertTagFunc mocks the InsertTag method.
InsertTagFunc func(ctx context.Context, t tag.Tag) error
@ -170,12 +170,12 @@ type TxMock struct {
// Doc is the doc argument value.
Doc document.Document
}
// InsertDocumentSearchJob holds details about calls to the InsertDocumentSearchJob method.
InsertDocumentSearchJob []struct {
// InsertSearchJob holds details about calls to the InsertSearchJob method.
InsertSearchJob []struct {
// Ctx is the ctx argument value.
Ctx context.Context
// Diff is the diff argument value.
Diff search.BlocksDifference
// Job is the job argument value.
Job search.Job
}
// InsertTag holds details about calls to the InsertTag method.
InsertTag []struct {
@ -216,7 +216,7 @@ type TxMock struct {
lockFetchOrganizationMembers sync.RWMutex
lockInsertDataSource sync.RWMutex
lockInsertDocument sync.RWMutex
lockInsertDocumentSearchJob sync.RWMutex
lockInsertSearchJob sync.RWMutex
lockInsertTag sync.RWMutex
lockRollback sync.RWMutex
lockUpdateOrganizationLogo sync.RWMutex
@ -538,42 +538,42 @@ func (mock *TxMock) InsertDocumentCalls() []struct {
return calls
}
// InsertDocumentSearchJob calls InsertDocumentSearchJobFunc.
func (mock *TxMock) InsertDocumentSearchJob(ctx context.Context, diff search.BlocksDifference) error {
// InsertSearchJob calls InsertSearchJobFunc.
func (mock *TxMock) InsertSearchJob(ctx context.Context, job search.Job) error {
callInfo := struct {
Ctx context.Context
Diff search.BlocksDifference
Ctx context.Context
Job search.Job
}{
Ctx: ctx,
Diff: diff,
Ctx: ctx,
Job: job,
}
mock.lockInsertDocumentSearchJob.Lock()
mock.calls.InsertDocumentSearchJob = append(mock.calls.InsertDocumentSearchJob, callInfo)
mock.lockInsertDocumentSearchJob.Unlock()
if mock.InsertDocumentSearchJobFunc == nil {
mock.lockInsertSearchJob.Lock()
mock.calls.InsertSearchJob = append(mock.calls.InsertSearchJob, callInfo)
mock.lockInsertSearchJob.Unlock()
if mock.InsertSearchJobFunc == nil {
var (
errOut error
)
return errOut
}
return mock.InsertDocumentSearchJobFunc(ctx, diff)
return mock.InsertSearchJobFunc(ctx, job)
}
// InsertDocumentSearchJobCalls gets all the calls that were made to InsertDocumentSearchJob.
// InsertSearchJobCalls gets all the calls that were made to InsertSearchJob.
// Check the length with:
//
// len(mockedTx.InsertDocumentSearchJobCalls())
func (mock *TxMock) InsertDocumentSearchJobCalls() []struct {
Ctx context.Context
Diff search.BlocksDifference
// len(mockedTx.InsertSearchJobCalls())
func (mock *TxMock) InsertSearchJobCalls() []struct {
Ctx context.Context
Job search.Job
} {
var calls []struct {
Ctx context.Context
Diff search.BlocksDifference
Ctx context.Context
Job search.Job
}
mock.lockInsertDocumentSearchJob.RLock()
calls = mock.calls.InsertDocumentSearchJob
mock.lockInsertDocumentSearchJob.RUnlock()
mock.lockInsertSearchJob.RLock()
calls = mock.calls.InsertSearchJob
mock.lockInsertSearchJob.RUnlock()
return calls
}

28
server/core/internal/server/internal/org/http.go
View File

@ -47,7 +47,7 @@ type Handler struct {
storer Storer
githubMan *github.Manager
webchangeClient *webchange.Client
searchJobs *search.Jobs
searchTrigger SearchTrigger
logoLocation string
}
@ -58,7 +58,7 @@ func NewHandler(
storer Storer,
githubMan *github.Manager,
webchangeClient *webchange.Client,
searchJobs *search.Jobs,
searchTrigger SearchTrigger,
logoLocationFormat string,
) *Handler {
return &Handler{
@ -67,7 +67,7 @@ func NewHandler(
storer: storer,
githubMan: githubMan,
webchangeClient: webchangeClient,
searchJobs: searchJobs,
searchTrigger: searchTrigger,
logoLocation: logoLocationFormat,
}
}
@ -151,7 +151,7 @@ func (h *Handler) InitializeOrganization(w http.ResponseWriter, r *http.Request)
return
}
if err = h.searchJobs.Enqueue(r.Context(), tx, search.BlocksDiff(nil, doc.Search())); err != nil {
if err = tx.InsertSearchJob(r.Context(), search.BranchScope(id, doc.ID, doc.BranchID)); err != nil {
httpserver.RespondError(h.log, w, err)
return
}
@ -162,6 +162,8 @@ func (h *Handler) InitializeOrganization(w http.ResponseWriter, r *http.Request)
return
}
h.searchTrigger.Trigger()
httpserver.Respond(
h.log,
w,
@ -291,9 +293,7 @@ func (h *Handler) TeardownOrganization(w http.ResponseWriter, r *http.Request) {
defer tx.Rollback() //nolint:errcheck // error provides no meaningful info
if err = h.searchJobs.Enqueue(r.Context(), tx, search.BlocksDifference{
RemovedOrganizations: []string{id},
}); err != nil {
if err = tx.InsertSearchJob(r.Context(), search.OrganizationScope(id)); err != nil {
httpserver.RespondError(h.log, w, err)
return
}
@ -314,6 +314,8 @@ func (h *Handler) TeardownOrganization(w http.ResponseWriter, r *http.Request) {
return
}
h.searchTrigger.Trigger()
// the logo is the organization's own object; the documents' files are
// left to the file manager, which reclaims them once the cascade nulls
// their foreign keys.
@ -364,6 +366,14 @@ type DB interface {
DBAgent
}
// SearchTrigger runs the search-job worker once a job has committed.
//
//go:generate ../../../../scripts/codegen/mock -t internal SearchTrigger search_trigger
type SearchTrigger interface {
// Trigger should run a search-job pass right away.
Trigger()
}
// Tx is an interface that combines sqlutil.Tx and DBAgent.
//
//go:generate ../../../../scripts/codegen/mock -t internal Tx tx
@ -382,8 +392,8 @@ type DBAgent interface {
// InsertDocument should insert the document.
InsertDocument(ctx context.Context, doc document.Document) error
// InsertDocumentSearchJob should insert the document search job.
InsertDocumentSearchJob(ctx context.Context, diff search.BlocksDifference) error
// InsertSearchJob should queue the search job's scope.
InsertSearchJob(ctx context.Context, job search.Job) error
// FetchDocumentHooksByOrganizationID should return every hook of the
// organization.

32
server/core/internal/server/internal/org/http_test.go
View File

@ -32,7 +32,7 @@ import (
)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
goleak.VerifyTestMain(m, testutil.IgnoreBleveWorkers())
}
// addSession stores a test session on the request context.
@ -89,7 +89,7 @@ func Test_NewHandler(t *testing.T) {
githubMan := &github.Manager{}
webchangeClient := &webchange.Client{}
searchJobs := search.NewJobs(true)
searchTrigger := &SearchTriggerMock{}
hdl := NewHandler(
slog.New(slog.DiscardHandler),
@ -97,7 +97,7 @@ func Test_NewHandler(t *testing.T) {
storer,
githubMan,
webchangeClient,
searchJobs,
searchTrigger,
"loc",
)
require.NotNil(t, hdl)
@ -106,7 +106,7 @@ func Test_NewHandler(t *testing.T) {
assert.Same(t, storer, hdl.storer)
assert.Same(t, githubMan, hdl.githubMan)
assert.Same(t, webchangeClient, hdl.webchangeClient)
assert.Same(t, searchJobs, hdl.searchJobs)
assert.Same(t, searchTrigger, hdl.searchTrigger)
assert.Equal(t, "loc", hdl.logoLocation)
}
@ -233,7 +233,7 @@ func Test_Handler_InitializeOrganization(t *testing.T) {
wasSearchJobInserted := func(count int) check {
return func(t *testing.T, _ *DBMock, tx *TxMock, _ *httptest.ResponseRecorder) {
assert.Len(t, tx.InsertDocumentSearchJobCalls(), count)
assert.Len(t, tx.InsertSearchJobCalls(), count)
}
}
@ -374,7 +374,7 @@ func Test_Handler_InitializeOrganization(t *testing.T) {
"Search job insertion error": {
Members: []string{"member1"},
Tx: &TxMock{
InsertDocumentSearchJobFunc: func(context.Context, search.BlocksDifference) error {
InsertSearchJobFunc: func(context.Context, search.Job) error {
return errors.New("boom")
},
},
@ -433,9 +433,9 @@ func Test_Handler_InitializeOrganization(t *testing.T) {
}
hdl := Handler{
log: slog.New(slog.DiscardHandler),
db: withTx(db, c.Tx, c.BeginErr),
searchJobs: search.NewJobs(true),
log: slog.New(slog.DiscardHandler),
db: withTx(db, c.Tx, c.BeginErr),
searchTrigger: &SearchTriggerMock{},
}
req := httptest.NewRequest(http.MethodPost, "http://test.com/", http.NoBody)
@ -811,14 +811,14 @@ func Test_Handler_TeardownOrganization(t *testing.T) {
wasSearchJobInserted := func(count int) check {
return func(t *testing.T, _ *DBMock, tx *TxMock, _ *StorerMock, _ *httptest.ResponseRecorder) {
ff := tx.InsertDocumentSearchJobCalls()
ff := tx.InsertSearchJobCalls()
require.Len(t, ff, count)
if count == 0 {
return
}
assert.Equal(t, []string{"org2"}, ff[0].Diff.RemovedOrganizations)
assert.Equal(t, search.OrganizationScope("org2"), ff[0].Job)
}
}
@ -928,7 +928,7 @@ func Test_Handler_TeardownOrganization(t *testing.T) {
"Search job insertion error": {
DB: &DBMock{},
Tx: &TxMock{
InsertDocumentSearchJobFunc: func(context.Context, search.BlocksDifference) error {
InsertSearchJobFunc: func(context.Context, search.Job) error {
return errors.New("boom")
},
},
@ -1014,10 +1014,10 @@ func Test_Handler_TeardownOrganization(t *testing.T) {
}
hdl := Handler{
log: slog.New(slog.DiscardHandler),
db: withTx(c.DB, c.Tx, c.BeginErr),
storer: storer,
searchJobs: search.NewJobs(true),
log: slog.New(slog.DiscardHandler),
db: withTx(c.DB, c.Tx, c.BeginErr),
storer: storer,
searchTrigger: &SearchTriggerMock{},
}
req := httptest.NewRequest(http.MethodPost, "http://test.com/", http.NoBody)

19
server/core/internal/server/server.go
View File

@ -19,7 +19,6 @@ import (
"github.com/oxynote/oxynote/server/core/internal/buildinfo"
datasourceCore "github.com/oxynote/oxynote/server/core/internal/datasource"
notificationCore "github.com/oxynote/oxynote/server/core/internal/notification"
"github.com/oxynote/oxynote/server/core/internal/search"
"github.com/oxynote/oxynote/server/core/internal/server/internal/assistant"
"github.com/oxynote/oxynote/server/core/internal/server/internal/auth"
"github.com/oxynote/oxynote/server/core/internal/server/internal/block"
@ -143,8 +142,8 @@ func NewServer(
githubMan *githubCore.Manager,
slackMan *slackCore.Manager,
webchangeClient *webchange.Client,
searchGateway document.SearchGateway,
searchJobs *search.Jobs,
searcher document.Searcher,
searchTrigger SearchTrigger,
notifier Notifier,
emailSender email.Sender,
client *http.Client,
@ -163,7 +162,6 @@ func NewServer(
Slack: slackMan.Configured(),
AIAssistant: opts.Assistant,
ChangeDetection: webchangeClient.Configured(),
Search: searchGateway.Configured(),
},
}
@ -174,10 +172,10 @@ func NewServer(
storageClient,
githubMan,
webchangeClient,
searchJobs,
searchTrigger,
opts.PublicURL+_organizationLogoLocation,
)
srv.handlers.document = document.NewHandler(log, db, githubMan, webchangeClient, searchGateway, searchJobs, notifier, storageClient)
srv.handlers.document = document.NewHandler(log, db, githubMan, webchangeClient, searcher, searchTrigger, notifier, storageClient)
srv.handlers.tag = tag.NewHandler(log, db)
srv.handlers.comment = comment.NewHandler(log, db, notifier)
srv.handlers.files = files.NewHandler(log, db, storageClient, opts.PublicURL)
@ -373,6 +371,15 @@ type Storer interface {
document.Storer
}
// SearchTrigger runs the search-job worker once a job has committed; the
// handlers call it after their commits.
//
//go:generate ../../scripts/codegen/mock -t internal SearchTrigger search_trigger
type SearchTrigger interface {
// Trigger should run a search-job pass right away.
Trigger()
}
// Notifier is an interface that combines notificationCore.Notifier and
// notificationCore.Publisher.
//

40
server/core/internal/server/server_test.go
View File

@ -17,7 +17,6 @@ import (
"github.com/oxynote/oxynote/server/core/internal/assistant/provider"
"github.com/oxynote/oxynote/server/core/internal/buildinfo"
datasourceCore "github.com/oxynote/oxynote/server/core/internal/datasource"
"github.com/oxynote/oxynote/server/core/internal/search"
"github.com/oxynote/oxynote/server/core/pkg/metricutil"
"github.com/oxynote/oxynote/server/core/pkg/testutil"
wsMock "github.com/oxynote/wetsocks/wsserver/_mock"
@ -60,17 +59,15 @@ func Test_Options_validate(t *testing.T) {
assert.EqualError(t, Options{}.validate(), "missing mcp session url")
}
// stubSearchGateway satisfies the document handler's SearchGateway with a
// fixed configured flag.
type stubSearchGateway struct {
configured bool
}
// stubSearcher satisfies the document handler's Searcher.
type stubSearcher struct{}
func (g stubSearchGateway) Configured() bool {
return g.configured
}
// stubSearchTrigger satisfies SearchTrigger with a no-op.
type stubSearchTrigger struct{}
func (stubSearchGateway) SearchDocuments(context.Context, string, string) ([]byte, error) {
func (stubSearchTrigger) Trigger() {}
func (stubSearcher) SearchDocuments(context.Context, string, string) ([]byte, error) {
return nil, nil
}
@ -83,7 +80,7 @@ func Test_NewServer(t *testing.T) {
githubMan, err := github.NewManager(nil, github.Options{})
require.NoError(t, err)
assistantMan := assistantCore.NewManager(log, nil, &redis.Pool{}, nil, nil, fc, nil, nil, search.NewJobs(false), nil, githubMan, nil, "claude")
assistantMan := assistantCore.NewManager(log, nil, &redis.Pool{}, nil, nil, fc, nil, nil, stubSearchTrigger{}, nil, githubMan, nil, "claude")
slackMan, err := slack.NewManager(log, nil, nil, nil, nil, slack.Options{})
require.NoError(t, err)
@ -107,8 +104,8 @@ func Test_NewServer(t *testing.T) {
githubMan,
slackMan,
webchange.NewClient("", ""),
stubSearchGateway{},
search.NewJobs(false),
stubSearcher{},
stubSearchTrigger{},
notifier,
nil,
http.DefaultClient,
@ -135,14 +132,14 @@ func Test_NewServer(t *testing.T) {
// a manager of this subtest's own: NewServer calls
// SetTreeNotifier, which must not race the parallel
// success case's identical call on a shared manager.
assistantCore.NewManager(log, nil, &redis.Pool{}, nil, nil, fc, nil, nil, search.NewJobs(false), nil, githubMan, nil, "claude"),
assistantCore.NewManager(log, nil, &redis.Pool{}, nil, nil, fc, nil, nil, stubSearchTrigger{}, nil, githubMan, nil, "claude"),
datasourceCore.NewManager(log, nil),
nil,
githubMan,
slackMan,
webchange.NewClient("", ""),
stubSearchGateway{},
search.NewJobs(false),
stubSearcher{},
stubSearchTrigger{},
notifier,
nil,
http.DefaultClient,
@ -175,8 +172,8 @@ func Test_NewServer(t *testing.T) {
githubMan,
slackMan,
webchange.NewClient("", ""),
stubSearchGateway{},
search.NewJobs(false),
stubSearcher{},
stubSearchTrigger{},
notifier,
nil,
http.DefaultClient,
@ -234,14 +231,14 @@ func Test_NewServer(t *testing.T) {
db,
fc,
storer,
assistantCore.NewManager(log, nil, &redis.Pool{}, nil, nil, fc, nil, nil, search.NewJobs(false), nil, githubMan, nil, "claude"),
assistantCore.NewManager(log, nil, &redis.Pool{}, nil, nil, fc, nil, nil, stubSearchTrigger{}, nil, githubMan, nil, "claude"),
datasourceCore.NewManager(log, nil),
nil,
githubMan,
slackMan,
webchange.NewClient("http://changedetection.test", ""),
stubSearchGateway{configured: true},
search.NewJobs(true),
stubSearcher{},
stubSearchTrigger{},
notifier,
nil,
http.DefaultClient,
@ -260,7 +257,6 @@ func Test_NewServer(t *testing.T) {
Model: "claude-opus-5",
},
ChangeDetection: true,
Search: true,
}, srv.capabilities)
})
}

8
server/core/pkg/testutil/testutil.go
View File

@ -21,8 +21,16 @@ import (
"github.com/jarcoal/httpmock"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
)
// IgnoreBleveWorkers ignores the analysis workers bleve starts when it is
// loaded and never stops, for goleak.VerifyTestMain in every package that
// links the search index.
func IgnoreBleveWorkers() goleak.Option {
return goleak.IgnoreTopFunction("github.com/blevesearch/bleve_index_api.AnalysisWorker")
}
// AddChiCtx adds the provided URL parameter to the context's chi route
// context, creating one when the context doesn't carry it yet.
func AddChiCtx(ctx context.Context, key, value string) context.Context {

6
server/core/pkg/testutil/testutil_test.go
View File

@ -187,3 +187,9 @@ func Test_Writer_Flush(t *testing.T) {
require.NoError(t, w.Flush())
assert.Equal(t, "data", b.String())
}
func Test_IgnoreBleveWorkers(t *testing.T) {
t.Parallel()
assert.NotNil(t, IgnoreBleveWorkers())
}

17
server/core/pkg/timeutil/periodic.go
View File

@ -13,6 +13,7 @@ type PeriodicExec struct {
fn func(ctx context.Context)
recovery func(v any)
immediate bool
trigger chan struct{}
}
// NewPeriodicExec creates a fresh periodic executor.
@ -35,6 +36,20 @@ func NewPeriodicExec(
fn: fn,
recovery: recovery,
immediate: immediate,
// one pending trigger is enough: a run serves every trigger that
// arrived before it, so later ones that find the slot taken are
// already covered.
trigger: make(chan struct{}, 1),
}
}
// Trigger runs the function at once, on top of the interval, which it
// does not reset. It never blocks and coalesces with a trigger already
// pending, so it is safe on a request path.
func (pe *PeriodicExec) Trigger() {
select {
case pe.trigger <- struct{}{}:
default:
}
}
@ -55,6 +70,8 @@ func (pe *PeriodicExec) Start(ctx context.Context) {
case <-tm.C:
pe.exec(ctx)
tm.Reset(pe.interval)
case <-pe.trigger:
pe.exec(ctx)
}
}
}

38
server/core/pkg/timeutil/periodic_test.go
View File

@ -22,6 +22,44 @@ func Test_NewPeriodicExec(t *testing.T) {
assert.NotNil(t, pe.fn)
assert.NotNil(t, pe.recovery)
assert.True(t, pe.immediate)
assert.Equal(t, 1, cap(pe.trigger))
}
func Test_PeriodicExec_Trigger(t *testing.T) {
t.Parallel()
// repeated triggers never block and collapse into one pending run.
pe := NewPeriodicExec(time.Hour, 0, func(context.Context) {}, nil, false)
pe.Trigger()
pe.Trigger()
assert.Len(t, pe.trigger, 1)
// a trigger runs the function without waiting for the interval, and
// the interval keeps its schedule.
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var runs int
stopCh := make(chan struct{})
pe = NewPeriodicExec(time.Hour, 0, func(context.Context) {
runs++
cancel()
}, nil, false)
go func() {
defer close(stopCh)
pe.Start(ctx)
}()
pe.Trigger()
<-stopCh
assert.Equal(t, 1, runs)
}
func Test_PeriodicExec_Start(t *testing.T) {

1
web/app/components/test-helpers.ts
View File

@ -78,7 +78,6 @@ export function seedCapabilities(capabilities: Partial<Capabilities> = {}) {
github: true,
slack: true,
changeDetection: true,
search: true,
aiAssistant: { status: AssistantStatus.Active, model: "test-model" },
...capabilities,
})

5
web/app/composables/api/useCapabilitiesAPI.nuxt.test.ts
View File

@ -17,7 +17,6 @@ function stubCapabilities(overrides: Partial<Capabilities> = {}): Capabilities {
github: true,
slack: true,
changeDetection: true,
search: true,
aiAssistant: { status: AssistantStatus.Active, model: "claude-opus-5" },
...overrides,
}
@ -52,7 +51,6 @@ describe("useCapabilitiesAPI", { concurrent: false }, () => {
github: true,
slack: false,
changeDetection: true,
search: false,
}),
)
const api = makeCapabilitiesAPI()
@ -62,7 +60,6 @@ describe("useCapabilitiesAPI", { concurrent: false }, () => {
expect(api.isGithubEnabled.value).toBe(true)
expect(api.isSlackEnabled.value).toBe(false)
expect(api.isChangeDetectionEnabled.value).toBe(true)
expect(api.isSearchEnabled.value).toBe(false)
})
it("reports every service enabled before the request resolves", ({
@ -73,7 +70,6 @@ describe("useCapabilitiesAPI", { concurrent: false }, () => {
expect(api.isGithubEnabled.value).toBe(true)
expect(api.isSlackEnabled.value).toBe(true)
expect(api.isChangeDetectionEnabled.value).toBe(true)
expect(api.isSearchEnabled.value).toBe(true)
expect(api.isAssistantEnabled.value).toBe(true)
})
@ -93,7 +89,6 @@ describe("useCapabilitiesAPI", { concurrent: false }, () => {
expect(api.isGithubEnabled.value).toBe(true)
expect(api.isSlackEnabled.value).toBe(true)
expect(api.isChangeDetectionEnabled.value).toBe(true)
expect(api.isSearchEnabled.value).toBe(true)
expect(api.isAssistantEnabled.value).toBe(true)
})
})

2
web/app/composables/api/useCapabilitiesAPI.ts
View File

@ -32,7 +32,6 @@ export default function () {
// leaves nothing for the apps settings section to show
const isAnyAppEnabled = enabled((c) => c.github || c.slack)
const isChangeDetectionEnabled = enabled((c) => c.changeDetection)
const isSearchEnabled = enabled((c) => c.search)
const assistantStatus = computed(
() => fetchCapabilities.data.value?.aiAssistant.status ?? null,
@ -52,7 +51,6 @@ export default function () {
isSlackEnabled,
isAnyAppEnabled,
isChangeDetectionEnabled,
isSearchEnabled,
isAssistantEnabled,
assistantStatus,
assistantModel,

1
web/app/utils/api/capabilities.ts
View File

@ -17,5 +17,4 @@ export interface Capabilities {
slack: boolean
aiAssistant: AssistantCapability
changeDetection: boolean
search: boolean
}