From 33d50e205c8b0fdca72d94e62516c9baa10cba69 Mon Sep 17 00:00:00 2001 From: david <58231312+davseby@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:40:46 +0300 Subject: [PATCH] remove rustfs in favour of filesystem storage --- README.md | 1 - docker/Caddyfile | 10 ---------- docker/docker-compose.dev.yaml | 33 +++++++-------------------------- docker/env/core.example.env | 23 +++++++++++++---------- e2e/docker-compose.yaml | 20 +------------------- server/CLAUDE.md | 3 +-- 6 files changed, 22 insertions(+), 68 deletions(-) diff --git a/README.md b/README.md index 9851a2b0..a15fa48e 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,6 @@ revoked in the app's settings. The dev stack exposes: - `8080` — front door (the app; APIs under `/auth-realtime` and `/core`) -- `8081` — RustFS console (at `/rustfs/console/`) - `8082` — changedetection.io - `8083` — Grafana - `8084` — Mailpit (dev email inbox) diff --git a/docker/Caddyfile b/docker/Caddyfile index 227fba53..686a6911 100644 --- a/docker/Caddyfile +++ b/docker/Caddyfile @@ -71,16 +71,6 @@ } } -:8081 { - import common - - # the console lives at /rustfs/console/, and the S3, STS and admin APIs - # it calls share this origin at "/" — so the root is left alone rather - # than redirected at the console, which would swallow ListBuckets and - # the console's own sign-in. - reverse_proxy rustfs:9001 -} - :8082 { import common diff --git a/docker/docker-compose.dev.yaml b/docker/docker-compose.dev.yaml index 65d5bc0e..a160504e 100644 --- a/docker/docker-compose.dev.yaml +++ b/docker/docker-compose.dev.yaml @@ -9,7 +9,6 @@ services: restart: unless-stopped ports: - "8080:8080" - - "8081:8081" - "8082:8082" - "8084:8084" logging: @@ -79,6 +78,12 @@ services: # docker/github/ and set OXYNOTE_CORE_GITHUB_APP_ID to enable the # GitHub App integration. - ./github:/etc/oxynote/core/github:ro + # uploaded files, which core keeps under OXYNOTE_CORE_STORAGE_PATH + # when no object store is configured. The mount is the home directory + # rather than the storage path itself: core runs as a non-root uid + # that owns only /home/nonroot in its image, and an empty volume + # takes the ownership of the directory it is mounted over. + - core_storage:/home/nonroot restart: unless-stopped logging: driver: local @@ -190,30 +195,6 @@ services: mode: non-blocking max-size: 10m max-file: "3" - rustfs: - container_name: rustfs - image: rustfs/rustfs:1.0.0-rc.3 - volumes: - - rustfs_data:/data - environment: - - RUSTFS_VOLUMES=/data - - RUSTFS_ADDRESS=0.0.0.0:9000 - - RUSTFS_CONSOLE_ENABLE=true - - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001 - - RUSTFS_ACCESS_KEY=sparta - - RUSTFS_SECRET_KEY=chad_warrior - restart: unless-stopped - healthcheck: - test: ["CMD", "curl", "-fsS", "http://127.0.0.1:9000/health"] - interval: 10s - timeout: 5s - retries: 5 - logging: - driver: local - options: - mode: non-blocking - max-size: 10m - max-file: "3" changedetection: container_name: changedetection image: dgtlmoon/changedetection.io:0.55.8 @@ -261,7 +242,7 @@ services: - GF_SECURITY_ADMIN_PASSWORD=admin volumes: - rustfs_data: + core_storage: mariadb_data: postgres_data: meilisearch_data: diff --git a/docker/env/core.example.env b/docker/env/core.example.env index ddd8ddeb..54067edc 100644 --- a/docker/env/core.example.env +++ b/docker/env/core.example.env @@ -17,20 +17,23 @@ OXYNOTE_CORE_VALKEY_ADDRESS=valkey:6379 # an empty URL disables the object store: objects are kept in a directory # on the local filesystem instead, and the rest of the STORAGE_ values are # unread. They are then not shared between instances and are lost with the -# container unless the storage path is a volume. -OXYNOTE_CORE_STORAGE_URL=http://rustfs:9000 +# container unless the storage path is a volume. The dev stack runs this +# way; set a URL and the values below to point it at an object store. +OXYNOTE_CORE_STORAGE_URL= # an empty region signs requests for us-east-1, which suits object stores # that have no concept of regions; set it when the bucket lives in a real # region that would reject a mismatched signature. OXYNOTE_CORE_STORAGE_REGION= -OXYNOTE_CORE_STORAGE_ACCESS_KEY=sparta -OXYNOTE_CORE_STORAGE_SECRET_KEY=chad_warrior -OXYNOTE_CORE_STORAGE_BUCKET=dev -# where objects are kept when no object store URL is configured. There is -# no default: with neither a URL nor a path, core refuses to boot rather -# than put objects somewhere nobody chose. The process must be able to -# create the directory, or write to it if it is already there. -OXYNOTE_CORE_STORAGE_PATH= +OXYNOTE_CORE_STORAGE_ACCESS_KEY= +OXYNOTE_CORE_STORAGE_SECRET_KEY= +OXYNOTE_CORE_STORAGE_BUCKET= +# where objects are kept when no object store URL is configured. Core has +# no default for it: with neither a URL nor a path it refuses to boot +# rather than put objects somewhere nobody chose. The process must be able +# to create the directory, or write to it if it is already there — in the +# core image /home/nonroot is the only directory owned by the uid it runs +# as, and the dev stack mounts a volume there. +OXYNOTE_CORE_STORAGE_PATH=/home/nonroot/storage OXYNOTE_CORE_SERVER_AUTH_BETTER_AUTH_URL=http://auth-realtime:8081/api/auth/get-session OXYNOTE_CORE_SERVER_MCP_SESSION_URL=http://auth-realtime:8081/api/internal/mcp/session OXYNOTE_CORE_AUTH_REALTIME_URL=http://auth-realtime:8081 diff --git a/e2e/docker-compose.yaml b/e2e/docker-compose.yaml index 275d84b5..745dd2da 100644 --- a/e2e/docker-compose.yaml +++ b/e2e/docker-compose.yaml @@ -108,10 +108,7 @@ services: - OXYNOTE_CORE_MEILISEARCH_MASTER_KEY=devmasterkey - OXYNOTE_CORE_VALKEY_NETWORK=tcp - OXYNOTE_CORE_VALKEY_ADDRESS=valkey:6379 - - OXYNOTE_CORE_STORAGE_URL=http://rustfs:9000 - - OXYNOTE_CORE_STORAGE_ACCESS_KEY=sparta - - OXYNOTE_CORE_STORAGE_SECRET_KEY=chad_warrior - - OXYNOTE_CORE_STORAGE_BUCKET=e2e + - OXYNOTE_CORE_STORAGE_PATH=/home/nonroot/storage - OXYNOTE_CORE_SERVER_AUTH_BETTER_AUTH_URL=http://auth-realtime:8081/api/auth/get-session # the MCP surface: core validates bearer tokens against auth-realtime # over the compose network, and the resource URL is the public @@ -133,8 +130,6 @@ services: condition: service_healthy meilisearch: condition: service_healthy - rustfs: - condition: service_healthy postgres: image: postgres:18.6-alpine @@ -167,19 +162,6 @@ services: timeout: 5s retries: 20 - rustfs: - image: rustfs/rustfs:1.0.0-rc.3 - environment: - - RUSTFS_VOLUMES=/data - - RUSTFS_ADDRESS=0.0.0.0:9000 - - RUSTFS_ACCESS_KEY=sparta - - RUSTFS_SECRET_KEY=chad_warrior - healthcheck: - test: ["CMD", "curl", "-fsS", "http://127.0.0.1:9000/health"] - interval: 5s - timeout: 5s - retries: 20 - # the tests read verification links out of mailpit's REST API, so its # http port is published rather than proxied through caddy. mailpit: diff --git a/server/CLAUDE.md b/server/CLAUDE.md index 910ff2fc..29f74afa 100644 --- a/server/CLAUDE.md +++ b/server/CLAUDE.md @@ -54,7 +54,6 @@ Go dependencies are fetched into the module cache at build time (`make deps` run `docker/Caddyfile` is the entrypoint for all client traffic: - `:8080` — front door. `/core/*` is path-stripped and reverse-proxied to `core:8080`; `/auth-realtime/*` is path-stripped and reverse-proxied to `auth-realtime:8081` (Better Auth, Hocuspocus, the merge proxy — the service itself still serves `/api/...` and `/hocuspocus`); everything else goes to the web SSR container (`web:3000`). -- `:8081` — RustFS console (served at `/rustfs/console/`; the root is the S3 API) - `:8082` — changedetection.io - `:8083` — Grafana (direct, not via Caddy) @@ -94,7 +93,7 @@ Every organization is seeded at initialization with a Prometheus data source nam ## External resource lifecycle (files, hooks) -Uploaded images live under `organizations/{org}/documents/{doc}/files/{blockUID}` — the file id **is** the image block's `uid` — and are tracked in `document_files`. That key is an object key in RustFS, or a path under the storage directory on a deployment running without an object store; `internal/storage` holds what the two backends share and `internal/storage/{s3,fs}` implement them. Hooks hold external resources too (changedetection.io watchers, GitHub tracking). Both are reclaimed by sweeps, not by request handlers: +Uploaded images live under `organizations/{org}/documents/{doc}/files/{blockUID}` — the file id **is** the image block's `uid` — and are tracked in `document_files`. That key is an object key in the S3-compatible store, or a path under the storage directory on a deployment running without one; `internal/storage` holds what the two backends share and `internal/storage/{s3,fs}` implement them. Hooks hold external resources too (changedetection.io watchers, GitHub tracking). Both are reclaimed by sweeps, not by request handlers: - **The row outlives its owner.** `document_files` and `document_hooks` reference documents, branches and organizations with `ON DELETE SET NULL`, so a deleted document — or an org deleted by Better Auth — leaves rows behind rather than destroying the only record of the external resource. `document_files.storage_key` stores the object key outright so it stays reachable once the FKs are gone. - **`internal/document/file/manager`** ticks every 5 minutes: it trims expired changelog snapshots, then reclaims files. A file is referenced if its id appears in any branch content of its document, any retained `document_branch_changelogs` snapshot, or any comment/reply — checked in SQL (`CheckDocumentFileReferenced`), so a spurious match only ever retains. Unreferenced files are stamped `unreferenced_at` and deleted a day later; rows with NULL FKs skip the wait; files younger than a day are never touched, which covers the gap between an upload and the first content persist that mentions it.