From 4df77fdadd7630efbdf464988c92c698669309e8 Mon Sep 17 00:00:00 2001 From: swithek <52840391+swithek@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:22:29 +0200 Subject: [PATCH] improve codeblock lang autodetection --- server/auth-realtime/src/schema/code-block.ts | 18 ++ .../code-block/CodeButtons.nuxt.test.ts | 145 +++++++++- .../editor/blocks/code-block/CodeButtons.vue | 128 ++++++--- .../blocks/code-block/detection.test.ts | 254 ++++++++++++++++++ .../editor/blocks/code-block/detection.ts | 184 +++++++++++++ .../editor/blocks/code-block/index.test.ts | 50 ++++ .../editor/blocks/code-block/index.ts | 23 ++ .../components/editor/diff/change-count.ts | 5 + 8 files changed, 756 insertions(+), 51 deletions(-) create mode 100644 web/app/components/editor/blocks/code-block/detection.test.ts create mode 100644 web/app/components/editor/blocks/code-block/detection.ts diff --git a/server/auth-realtime/src/schema/code-block.ts b/server/auth-realtime/src/schema/code-block.ts index 3b39e3dd..41911797 100644 --- a/server/auth-realtime/src/schema/code-block.ts +++ b/server/auth-realtime/src/schema/code-block.ts @@ -73,6 +73,24 @@ export const CodeBlock = CodeBlockLowlight.extend({ ], ] }, + addAttributes() { + return { + ...this.parent?.(), + auto: { + default: false, + parseHTML: (element) => + element.getAttribute("data-auto") === + "true", + renderHTML: (attrs) => { + if (!attrs.auto) { + return {} + } + + return { "data-auto": "true" } + }, + }, + } + }, addOptions(): CodeBlockOptions { const parent = this.parent ? this.parent() : null return { diff --git a/web/app/components/editor/blocks/code-block/CodeButtons.nuxt.test.ts b/web/app/components/editor/blocks/code-block/CodeButtons.nuxt.test.ts index eb4a9acb..4e8902a5 100644 --- a/web/app/components/editor/blocks/code-block/CodeButtons.nuxt.test.ts +++ b/web/app/components/editor/blocks/code-block/CodeButtons.nuxt.test.ts @@ -2,7 +2,9 @@ import { flushPromises, type VueWrapper } from "@vue/test-utils" import { beforeEach, describe, it, vi } from "vitest" import CodeButtons from "./CodeButtons.vue" import { + commandArgs, makeEditor, + makeEditorState, makeNode, mountNodeView, type EditorStub, @@ -14,6 +16,7 @@ const GO_SNIPPET = 'package main\n\nimport "fmt"\n\nfunc main() {\n\tfmt.Println("hi")\n}\n' const COPIED_RESET_MS = 700 +const STORE_DELAY_MS = 500 function mountButtons( options: { @@ -110,7 +113,27 @@ describe("", { concurrent: false }, () => { emitFrom(wrapper, Select, "update:modelValue", "rust") expect(updateAttributes).toHaveBeenCalledTimes(1) - expect(updateAttributes).toHaveBeenCalledWith({ language: "rust" }) + expect(updateAttributes).toHaveBeenCalledWith({ + language: "rust", + auto: false, + }) + }) + + it("turns detection off when the user picks the detected language", async ({ + expect, + }) => { + const updateAttributes = vi.fn() + const wrapper = await mountButtons({ + attrs: { language: "rust", auto: true }, + updateAttributes: updateAttributes, + }) + + emitFrom(wrapper, Select, "update:modelValue", "rust") + + expect(updateAttributes).toHaveBeenCalledWith({ + language: "rust", + auto: false, + }) }) it("stores nothing when the picked language is already set", async ({ @@ -142,9 +165,7 @@ describe("", { concurrent: false }, () => { ), }) - editor.on.mock.calls.forEach(([, handler]) => { - ;(handler as () => void)() - }) + reportUpdate(editor) await nextTick() expect(editor.on).toHaveBeenCalledTimes(1) @@ -161,14 +182,95 @@ describe("", { concurrent: false }, () => { editor: editor, }) - editor.on.mock.calls.forEach(([, handler]) => { - ;(handler as () => void)() - }) + reportUpdate(editor) await nextTick() expect(selectedLanguage(wrapper)).toBe("Python") }) + it("stores the detected language once the user stops typing", async ({ + expect, + }) => { + const editor = makeEditor() + await mountEdited(editor) + + reportUpdate(editor) + await vi.advanceTimersByTimeAsync(STORE_DELAY_MS - 1) + + expect(commandArgs(editor.commands, "command")).toBeUndefined() + + await vi.advanceTimersByTimeAsync(1) + + const tr = { setNodeAttribute: vi.fn(), setMeta: vi.fn() } + const [store] = commandArgs(editor.commands, "command") as [ + (props: { tr: unknown }) => boolean, + ] + store({ tr: tr }) + + expect(tr.setNodeAttribute.mock.calls).toEqual([ + [0, "language", "go"], + [0, "auto", true], + ]) + expect(tr.setMeta).toHaveBeenCalledWith("addToHistory", false) + }) + + it("stores nothing for a remote change", async ({ expect }) => { + const editor = makeEditor() + await mountEdited(editor) + + reportUpdate(editor, true) + await vi.advanceTimersByTimeAsync(STORE_DELAY_MS) + + expect(commandArgs(editor.commands, "command")).toBeUndefined() + }) + + it("stores nothing while the caret is outside the block", async ({ + expect, + }) => { + const editor = makeEditor({ + state: makeEditorState({ from: 10, to: 10 }), + }) + await mountEdited(editor) + + reportUpdate(editor) + await vi.advanceTimersByTimeAsync(STORE_DELAY_MS) + + expect(commandArgs(editor.commands, "command")).toBeUndefined() + }) + + it("stores nothing over a language the user picked", async ({ expect }) => { + const editor = makeEditor() + await mountEdited(editor, { language: "python" }) + + reportUpdate(editor) + await vi.advanceTimersByTimeAsync(STORE_DELAY_MS) + + expect(commandArgs(editor.commands, "command")).toBeUndefined() + }) + + it("stores nothing when the detected language is already stored", async ({ + expect, + }) => { + const editor = makeEditor() + await mountEdited(editor, { language: "go", auto: true }) + + reportUpdate(editor) + await vi.advanceTimersByTimeAsync(STORE_DELAY_MS) + + expect(commandArgs(editor.commands, "command")).toBeUndefined() + }) + + it("stores nothing once unmounted", async ({ expect }) => { + const editor = makeEditor() + const wrapper = await mountEdited(editor) + + reportUpdate(editor) + wrapper.unmount() + await vi.advanceTimersByTimeAsync(STORE_DELAY_MS) + + expect(commandArgs(editor.commands, "command")).toBeUndefined() + }) + it("stops listening for editor updates once unmounted", async ({ expect, }) => { @@ -285,3 +387,32 @@ describe("", { concurrent: false }, () => { ) }) }) + +// run the editor's update listeners as a change would. A remote change +// carries the sync plugin's meta, a local one carries none. +function reportUpdate(editor: EditorStub, remote = false) { + const transaction = { getMeta: () => (remote ? {} : undefined) } + + editor.on.mock.calls.forEach(([, handler]) => { + ;(handler as (props: { transaction: unknown }) => void)({ + transaction: transaction, + }) + }) +} + +// mount an empty block, then give it the Go snippet as an edit would +async function mountEdited( + editor: EditorStub, + attrs: Record = {}, +) { + const wrapper = await mountButtons({ attrs: attrs, editor: editor }) + await wrapper.setProps({ + node: makeNode( + { uid: "code-1", language: null, ...attrs }, + { textContent: GO_SNIPPET }, + ), + }) + vi.useFakeTimers() + + return wrapper +} diff --git a/web/app/components/editor/blocks/code-block/CodeButtons.vue b/web/app/components/editor/blocks/code-block/CodeButtons.vue index 01cafca9..37458360 100644 --- a/web/app/components/editor/blocks/code-block/CodeButtons.vue +++ b/web/app/components/editor/blocks/code-block/CodeButtons.vue @@ -5,12 +5,20 @@ import { defaultExtendedCodeBlockLanguage, extendedCodeBlockLanguageOptions, } from "./languages" -import { lowlight, type CodeBlockOptions } from "./index" +import { isLanguageDetected, type CodeBlockOptions } from "./index" +import { detectLanguage } from "./detection" +import { isChangeOrigin } from "@tiptap/extension-collaboration" +import type { Transaction } from "@tiptap/pm/state" + +// detection runs every grammar over the code, so it waits until the code +// stops changing +const STORE_DELAY_MS = 500 const props = defineProps({ ...nodeViewProps, }) const { isEditable } = useEditorMeta() + const isCopied = ref(false) const typeClass = computed(() => { return (props.extension.options as CodeBlockOptions).type === "comment" @@ -20,48 +28,9 @@ const typeClass = computed(() => { const detectedLanguage = ref(defaultExtendedCodeBlockLanguage) -function detectLanguage() { - if (props.node.attrs.language) { - return - } - - const content = props.node.textContent - if (!content.trim()) { - detectedLanguage.value = defaultExtendedCodeBlockLanguage - return - } - - try { - const result = lowlight.highlightAuto(content) - - if ( - result.data?.language && - result.data.relevance && - result.data.relevance >= 2 - ) { - detectedLanguage.value = result.data.language - return - } - } catch { - // Ignore errors - } - - detectedLanguage.value = defaultExtendedCodeBlockLanguage -} - -onBeforeMount(() => { - detectLanguage() -}) -onMounted(() => { - props.editor.on("update", detectLanguage) -}) -onUnmounted(() => { - props.editor.off("update", detectLanguage) -}) - const currentLang = computed({ get: () => { - // If language is null (auto mode), show detected language + // a block the detector has not stored yet shows its own guess if (!props.node.attrs.language) { return detectedLanguage.value } @@ -69,13 +38,84 @@ const currentLang = computed({ return props.node.attrs.language as string }, set: (lang: string) => { - // User selected a language - set it (no longer auto) - if (props.node.attrs.language !== lang) { - props.updateAttributes({ language: lang }) + // picking a language turns detection off, even when it is the + // detected one + if (props.node.attrs.language !== lang || props.node.attrs.auto) { + props.updateAttributes({ language: lang, auto: false }) } }, }) +let storeTimeout: ReturnType | undefined + +onBeforeMount(() => { + updateDetectedLanguage() +}) +onMounted(() => { + props.editor.on("update", onEditorUpdate) +}) +onUnmounted(() => { + props.editor.off("update", onEditorUpdate) + clearTimeout(storeTimeout) +}) + +function updateDetectedLanguage() { + if (props.node.attrs.language) { + return + } + + detectedLanguage.value = detectLanguage(props.node.textContent) +} + +function onEditorUpdate({ transaction }: { transaction: Transaction }) { + updateDetectedLanguage() + + // only the user typing in the block stores its language, because the + // server counts every write as an edit by its user + if (isChangeOrigin(transaction) || !isCaretInside()) { + return + } + + clearTimeout(storeTimeout) + storeTimeout = setTimeout(storeDetectedLanguage, STORE_DELAY_MS) +} + +function isCaretInside(): boolean { + const pos = props.getPos() + if (typeof pos !== "number") { + return false + } + + const { from, to } = props.editor.state.selection + + return from >= pos && to <= pos + props.node.nodeSize +} + +function storeDetectedLanguage() { + const pos = props.getPos() + if (typeof pos !== "number" || !isLanguageDetected(props.node)) { + return + } + + const language = detectLanguage(props.node.textContent) + if (props.node.attrs.language === language && props.node.attrs.auto) { + return + } + + // the write stays out of the undo history. A tracked write after an + // undo would clear what can be redone. + props.editor + .chain() + .command(({ tr }) => { + tr.setNodeAttribute(pos, "language", language) + tr.setNodeAttribute(pos, "auto", true) + tr.setMeta("addToHistory", false) + + return true + }) + .run() +} + async function copyCodeBlock() { try { await navigator.clipboard.writeText(props.node.textContent || "") diff --git a/web/app/components/editor/blocks/code-block/detection.test.ts b/web/app/components/editor/blocks/code-block/detection.test.ts new file mode 100644 index 00000000..4f948ced --- /dev/null +++ b/web/app/components/editor/blocks/code-block/detection.test.ts @@ -0,0 +1,254 @@ +import { describe, it } from "vitest" +import { detectLanguage } from "./detection" + +const SNIPPETS = [ + { + name: 'detects bash in a snippet starting "#!/usr/bin/env bash"', + input: + '#!/usr/bin/env bash\nset -euo pipefail\n\nfor f in *.log; do\n echo "processing $f"\n gzip "$f"\ndone', + expected: "bash", + }, + { + name: 'detects bash in a snippet starting "docker compose up -d"', + input: "docker compose up -d\nmake e2e-dev\npnpm install", + expected: "bash", + }, + { + name: 'detects c in a snippet starting "#include "', + input: + '#include \n\nint main(void) {\n printf("hello\\n");\n return 0;\n}', + expected: "c", + }, + { + name: 'detects cpp in a snippet starting "#include "', + input: + "#include \n#include \n\nint main() {\n std::vector v = {1, 2, 3};\n for (auto x : v) {\n std::cout << x << std::endl;\n }\n return 0;\n}", + expected: "cpp", + }, + { + name: 'detects csharp in a snippet starting "using System;"', + input: + "using System;\nusing System.Collections.Generic;\n\nnamespace App\n{\n public class Program\n {\n public static void Main(string[] args)\n {\n var items = new List();\n foreach (var item in items)\n {\n Console.WriteLine(item);\n }\n }\n }\n}", + expected: "csharp", + }, + { + name: 'detects csharp in a snippet starting "public class User"', + input: + 'public class User\n{\n public string Id { get; set; }\n public string Name { get; set; }\n\n public async Task LoadAsync(HttpClient client)\n {\n var json = await client.GetStringAsync("/me");\n return JsonSerializer.Deserialize(json);\n }\n}', + expected: "csharp", + }, + { + name: 'detects css in a snippet starting ".button {"', + input: + ".button {\n color: red;\n padding: 0.5rem 1rem;\n}\n\n.button:hover {\n color: blue;\n}", + expected: "css", + }, + { + name: 'detects curl in a snippet starting "curl -X POST https://api.example"', + input: + 'curl -X POST https://api.example.com/v1/items \\\n -H "Authorization: Bearer $TOKEN" \\\n -H "Content-Type: application/json" \\\n -d \'{"name": "x"}\'', + expected: "curl", + }, + { + name: 'detects diff in a snippet starting "--- a/file.txt"', + input: + "--- a/file.txt\n+++ b/file.txt\n@@ -1,3 +1,3 @@\n-old line\n+new line\n context", + expected: "diff", + }, + { + name: 'detects go in a snippet starting "package main"', + input: + 'package main\n\nimport (\n\t"fmt"\n\t"net/http"\n)\n\ntype Server struct {\n\taddr string\n}\n\nfunc (s *Server) Run() error {\n\tmux := http.NewServeMux()\n\tmux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {\n\t\tfmt.Fprintln(w, "hi")\n\t})\n\n\tif err := http.ListenAndServe(s.addr, mux); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}', + expected: "go", + }, + { + name: 'detects go in a snippet starting "func Sum(values []int) int {"', + input: + "func Sum(values []int) int {\n\ttotal := 0\n\tfor _, v := range values {\n\t\ttotal += v\n\t}\n\n\treturn total\n}", + expected: "go", + }, + { + name: 'detects go in a snippet starting "type User struct {"', + input: + 'type User struct {\n\tID string `json:"id"`\n\tName string `json:"name"`\n\tCreatedAt time.Time `json:"created_at"`\n}', + expected: "go", + }, + { + name: 'detects graphql in a snippet starting "query GetUser($id: ID!) {"', + input: + "query GetUser($id: ID!) {\n user(id: $id) {\n id\n name\n posts(first: 10) {\n title\n }\n }\n}", + expected: "graphql", + }, + { + name: 'detects ini in a snippet starting "[server]"', + input: + "[server]\nhost = 0.0.0.0\nport = 8080\n\n[database]\nurl = postgres://localhost/db", + expected: "ini", + }, + { + name: 'detects java in a snippet starting "import java.util.List;"', + input: + 'import java.util.List;\n\npublic class Main {\n public static void main(String[] args) {\n List items = List.of("a", "b");\n for (String item : items) {\n System.out.println(item);\n }\n }\n}', + expected: "java", + }, + { + name: 'detects javascript in a snippet starting "const express = require(\\"express"', + input: + 'const express = require("express")\nconst app = express()\n\napp.get("/", (req, res) => {\n\tres.send("hello")\n})\n\napp.listen(3000, () => console.log("up"))', + expected: "javascript", + }, + { + name: 'detects json in a snippet starting "{"', + input: + '{\n "name": "oxynote",\n "version": "1.0.0",\n "private": true,\n "scripts": {\n "dev": "nuxt dev",\n "build": "nuxt build"\n },\n "tags": ["a", "b"],\n "count": 3\n}', + expected: "json", + }, + { + name: 'detects json in a snippet starting "{\\"id\\": 12, \\"ok\\": true}"', + input: '{"id": 12, "ok": true}', + expected: "json", + }, + { + name: 'detects json in a snippet starting "["', + input: '[\n { "id": 1, "name": "a" },\n { "id": 2, "name": "b" }\n]', + expected: "json", + }, + { + name: 'detects kotlin in a snippet starting "fun main() {"', + input: + 'fun main() {\n val items = listOf("a", "b")\n for (item in items) {\n println(item)\n }\n}', + expected: "kotlin", + }, + { + name: 'detects makefile in a snippet starting "CC = gcc"', + input: + "CC = gcc\nCFLAGS = -Wall -O2\n\napp: main.o util.o\n\t$(CC) $(CFLAGS) -o $@ $^\n\nclean:\n\trm -f *.o app", + expected: "makefile", + }, + { + name: 'detects markdown in a snippet starting "# Title"', + input: + "# Title\n\nSome **bold** text and a [link](https://example.com).\n\n- one\n- two", + expected: "markdown", + }, + { + name: 'detects php in a snippet starting " int:\n result = 0\n for v in values:\n result += v\n return result\n\nif __name__ == "__main__":\n print(total([1, 2, 3]))', + expected: "python", + }, + { + name: 'detects ruby in a snippet starting "class User"', + input: + 'class User\n attr_reader :name\n\n def initialize(name)\n @name = name\n end\n\n def greet\n puts "hi #{@name}"\n end\nend', + expected: "ruby", + }, + { + name: 'detects rust in a snippet starting "use std::collections::HashMap;"', + input: + 'use std::collections::HashMap;\n\nfn main() {\n let mut map: HashMap = HashMap::new();\n map.insert("a".to_string(), 1);\n for (k, v) in &map {\n println!("{k}: {v}");\n }\n}', + expected: "rust", + }, + { + name: 'detects shell in a snippet starting "$ docker compose up -d"', + input: "$ docker compose up -d\n$ make e2e-dev", + expected: "shell", + }, + { + name: 'detects sql in a snippet starting "SELECT u.id, u.name, count(o.id)"', + input: + "SELECT u.id, u.name, count(o.id) AS orders\nFROM users u\nLEFT JOIN orders o ON o.user_id = u.id\nWHERE u.active = true\nGROUP BY u.id\nORDER BY orders DESC;", + expected: "sql", + }, + { + name: 'detects swift in a snippet starting "import Foundation"', + input: + 'import Foundation\n\nstruct User {\n let name: String\n}\n\nfunc greet(_ user: User) -> String {\n return "hi \\(user.name)"\n}', + expected: "swift", + }, + { + name: 'detects typescript in a snippet starting "interface User {"', + input: + 'interface User {\n\tid: string\n\tname: string\n}\n\nexport async function loadUser(id: string): Promise {\n\tconst res = await fetch(`/api/users/${id}`)\n\tif (!res.ok) {\n\t\tthrow new Error("failed")\n\t}\n\n\treturn (await res.json()) as User\n}', + expected: "typescript", + }, + { + name: 'detects xml in a snippet starting ""', + input: + '\n\n x\n \n
hi
\n \n', + expected: "xml", + }, + { + name: 'detects yaml in a snippet starting "services:"', + input: + 'services:\n web:\n image: nginx:latest\n ports:\n - "8080:80"\n environment:\n - KEY=value', + expected: "yaml", + }, +] + +// the highlighter's keyword count alone names another language for this one +const GO_WITH_SHARED_KEYWORDS = + 'var (\n\tErrNotFound = errors.New("not found")\n\tErrInvalid = errors.New("invalid")\n)\n\ntype Store interface {\n\tGet(ctx context.Context, id string) (User, error)\n\tPut(ctx context.Context, u User) error\n}' + +const NO_CLEAR_LEAD = + 'public := os.Getenv("PUBLIC_URL")\nvar client = &http.Client{Timeout: 5 * time.Second}\nvar result string\nif public == "" {\n\tresult = "none"\n}\nnew := strings.TrimSpace(result)' + +const WEAK_EVIDENCE = + 'sum(rate(http_requests_total{job="api"}[5m])) by (status)' + +describe("detectLanguage", () => { + it.for(SNIPPETS)("$name", ({ input, expected }, { expect }) => { + expect(detectLanguage(input)).toBe(expected) + }) + + it("detects go where the shared keywords favour another language", ({ + expect, + }) => { + expect(detectLanguage(GO_WITH_SHARED_KEYWORDS)).toBe("go") + }) + + it("detects json written on one line", ({ expect }) => { + expect(detectLanguage('{"id": 12, "ok": true}')).toBe("json") + }) + + it("falls back to plaintext for blank code", ({ expect }) => { + expect(detectLanguage(" \n ")).toBe("plaintext") + }) + + it("falls back to plaintext when no language has a clear lead", ({ + expect, + }) => { + expect(detectLanguage(NO_CLEAR_LEAD)).toBe("plaintext") + }) + + it("falls back to plaintext when the evidence is weak", ({ expect }) => { + expect(detectLanguage(WEAK_EVIDENCE)).toBe("plaintext") + }) +}) diff --git a/web/app/components/editor/blocks/code-block/detection.ts b/web/app/components/editor/blocks/code-block/detection.ts new file mode 100644 index 00000000..fda2f3db --- /dev/null +++ b/web/app/components/editor/blocks/code-block/detection.ts @@ -0,0 +1,184 @@ +import { lowlight } from "./index" +import { defaultExtendedCodeBlockLanguage } from "./languages" + +// one matching pattern counts as much as this many keyword matches +const PATTERN_WEIGHT = 5 + +// the best language needs this score and this lead over the next one. A +// block that reaches neither stays plaintext. +const MIN_SCORE = 5 +const MIN_LEAD = 3 + +// constructs only one of the languages writes. The highlighter scores a +// language by its keywords alone, and many languages share those. +const LANGUAGE_PATTERNS: Record = { + bash: [ + /^#!.*\b(ba|z)?sh\b/, + /^\s*(sudo |cd |export \w+=|echo |source )/m, + /^\s*(git|docker|kubectl|helm|npm|pnpm|yarn|npx|make|brew|apt(-get)?|pip3?|cargo|go|terraform|aws|gcloud|ssh|scp|wget|tar|mkdir|chmod|cat|grep|ls|rm|cp|mv) [\w.\-/]/m, + /\b(then|fi|done|esac)\s*$/m, + /\$\{?\w+\}?"|\$\(/, + ], + c: [ + /^#include <\w+\.h>/m, + /\b(printf|malloc|sizeof)\(/, + /^(int|void|char) \w+\([^)]*\)\s*\{/m, + ], + cpp: [ + /^#include <[a-z_]+>/m, + /\bstd::/, + /^(template|namespace|class) .*[<{]/m, + /\b(cout|cin|endl)\b/, + ], + csharp: [ + /^using [\w.]+;/m, + /^namespace [\w.]+\s*[{;]?$/m, + /\{ get; (set|init); \}/, + /\b(Console\.Write|string\[\] args|async Task|foreach \(|IEnumerable<)/, + /\.(Where|Select|OrderBy|ToList)\(/, + /\b(Task<|IActionResult|public record |private readonly )/, + /^\s*\[(Http\w+|ApiController|Route|Fact|Test)\b/m, + ], + css: [ + /^[.#][\w-]+[^\n{};=(]*\{\s*$/m, + /^(html|body|a|p|div|span|h[1-6]|ul|li|table|input|button|img)\b[^\n{};=(]*\{\s*$/m, + /^\s*[a-z-]+:\s*[^;{}\n]+;\s*$/m, + /(^|\s)(@media|@import|:root|!important)\b/m, + ], + curl: [/^\s*curl\s/], + diff: [ + /^diff --git /m, + /^@@ -\d+(,\d+)? \+\d+(,\d+)? @@/m, + /^--- .*\n\+\+\+ /m, + ], + go: [ + /^package \w+\s*$/m, + /^func (\(\w+ \*?[\w.]+\) )?\w+(\[[^\]]+\])?\(/m, + /\w+(, \w+)* := /, + /^type \w+ (struct|interface) \{/m, + /\berr [!=]= nil\b|\bfunc\(|\bdefer |\bchan |\brange /, + /^import \(\s*$|^import "[\w./-]+"$/m, + ], + graphql: [ + /^\s*(query|mutation|subscription|fragment)\b[^=;\n]*\{\s*$/m, + /^\s*\w+(\([^)]*\))?:\s*\[?\w+!?\]?!?\s*$/m, + /\$\w+:\s*\[?\w+!?\]?/, + ], + ini: [/^\[[\w.\- ]+\]\s*$/m, /^[\w.-]+ = \S/m], + java: [ + /^import (static )?(java|javax|org|com)\.[\w.*]+;/m, + /\bSystem\.(out|err)\.|String\[\] args/, + /^\s*@(Override|Autowired|RestController|GetMapping|PostMapping|Test|Entity)\b/m, + /\b(public|private|protected) (static )?(final )?[\w<>[\]]+ \w+\([^)]*\)\s*(throws \w+\s*)?\{/, + ], + javascript: [ + /\b(const|let) \w+ = /, + /=>\s*[{(\w]/, + /\b(require\(|console\.\w+\(|document\.|module\.exports|window\.)/, + /\b(async )?function\*? \w*\(/, + /^(import|export) .* from ["']/m, + ], + json: [ + /^\s*\{\s*("[^"\n]*"\s*:|\}\s*$)/, + /^\s*\[\s*([{["\d\]-]|true|false|null)/, + /^\s*"[^"\n]*"\s*:\s*("|\d|true|false|null|\{|\[)/m, + ], + kotlin: [ + /\bfun \w+\(/, + /\b(val|var) \w+(: \w+)? = /, + /\b(println\(|listOf\(|data class |companion object)/, + ], + makefile: [/^\.PHONY:/m, /^[\w.\-/%]+:[^=\n]*\n\t\S/m, /\$\((\w+)\)|\$@|\$\w+|\$this\b/, /\bfunction \w+\([^)]*\$\w+/], + python: [ + /^\s*(def|class) \w+[^\n{;]*:\s*(#.*)?$/m, + /^from [\w.]+ import [\w*, ]+$/m, + /^import \w+(\.\w+)*( as \w+)?$/m, + /\b(self\.\w+|__\w+__|print\(|elif |None\b|True\b|False\b)/, + /^\s*(if|for|while|with|try|except)\b[^\n{;]*:\s*$/m, + ], + ruby: [ + /^\s*def \w+[^\n:{]*$/m, + /^\s*end\s*$/m, + /\b(attr_(reader|writer|accessor)|puts |require ')/, + /\bdo \|[\w, ]+\|/, + /@\w+ = /, + ], + rust: [ + /^use \w+(::[\w{}*, ]+)+;/m, + /\bfn \w+(<[^>]+>)?\(/, + /\blet (mut )?\w+/, + /^\s*#\[\w+/m, + /\b(impl|pub (fn|struct|enum)|println!|Some\(|Ok\(|&str\b)/, + ], + shell: [/^\$ \S/m], + sql: [ + /^\s*(select)\b[\s\S]*\bfrom\b/im, + /^\s*(insert into|delete from|create (table|index|view)|alter table|drop table)\b/im, + /^\s*update \w+ set\b/im, + /\b(where|group by|order by|left join|inner join|primary key|not null)\b/i, + ], + swift: [ + /^import (Foundation|UIKit|SwiftUI)\b/m, + /\bfunc \w+\([^)]*\) -> /, + /\b(guard let|if let|let \w+: \w+)/, + /\\\(\w+/, + ], + typescript: [ + /\b(interface|type) \w+(<[^>]+>)? (=|\{|extends)/, + /[\w)\]]\??: (string|number|boolean|unknown|void|any|Promise<|Record<|\w+\[\])/, + /\b(as (const|unknown|string|number|\w+)\b|import type |readonly |enum \w+ \{)/, + /\bexport (default |async )?(function|const|class|interface|type)\b/, + ], + xml: [ + /^\s*<(!DOCTYPE|\?xml|[a-zA-Z][\w:-]*)(\s[^<>]*)?\/?>/, + /<\/[a-zA-Z][\w:-]*>\s*$/m, + /<[a-zA-Z][\w:-]*(\s+[\w:-]+="[^"]*")+\s*\/?>/, + ], + yaml: [/^[\w.-]+:\s*$/m, /^\s+- [\w"'{[]/m, /^\s+[\w.-]+: \S/m, /^---\s*$/m], +} + +export function detectLanguage(code: string): string { + if (!code.trim()) { + return defaultExtendedCodeBlockLanguage + } + + let bestLanguage = defaultExtendedCodeBlockLanguage + let bestScore = 0 + let nextScore = 0 + + for (const [language, patterns] of Object.entries(LANGUAGE_PATTERNS)) { + const score = languageScore(language, patterns, code) + + if (score > bestScore) { + nextScore = bestScore + bestScore = score + bestLanguage = language + } else if (score > nextScore) { + nextScore = score + } + } + + if (bestScore < MIN_SCORE || bestScore - nextScore < MIN_LEAD) { + return defaultExtendedCodeBlockLanguage + } + + return bestLanguage +} + +function languageScore( + language: string, + patterns: RegExp[], + code: string, +): number { + const relevance = lowlight.highlight(language, code).data?.relevance ?? 0 + const matches = patterns.filter((pattern) => pattern.test(code)).length + + return relevance + matches * PATTERN_WEIGHT +} diff --git a/web/app/components/editor/blocks/code-block/index.test.ts b/web/app/components/editor/blocks/code-block/index.test.ts index 6f47138f..6c9b6ece 100644 --- a/web/app/components/editor/blocks/code-block/index.test.ts +++ b/web/app/components/editor/blocks/code-block/index.test.ts @@ -12,6 +12,7 @@ import type { CodeBlockOptions } from "./index" import { CodeBlock, CodeBlockTitle, + isLanguageDetected, lowlight, setUpCodeBlockNode, TitledCodeBlock, @@ -106,6 +107,17 @@ function rightSide(...children: PMNode[]): PMNode { const docOf = docBuilder(schema) +const detectionSchema = new Schema({ + nodes: { + doc: { content: `${CODE_BLOCK_NAME}+` }, + [CODE_BLOCK_NAME]: { + content: "text*", + attrs: { language: { default: null }, auto: { default: false } }, + }, + text: {}, + }, +}) + describe("lowlight", () => { it("registers curl alongside the common grammars", ({ expect }) => { expect(lowlight.listLanguages()).toContain("curl") @@ -393,6 +405,22 @@ describe("CodeBlock", () => { }) }) + describe("addAttributes", () => { + it("leaves detection off for a block that does not name it", ({ + expect, + }) => { + const addAttributes = getExtensionField< + () => Record + >( + CodeBlock, + "addAttributes", + extensionContext(CodeBlock, CODE_BLOCK_NAME), + ) + + expect(addAttributes().auto?.default).toBe(false) + }) + }) + describe("addKeyboardShortcuts", () => { it("keeps the inherited shortcuts alongside its own", ({ expect }) => { const run = shortcutsAt(CodeBlock, CODE_BLOCK_NAME, docOf(code("ab")), 1) @@ -731,3 +759,25 @@ describe("setUpCodeBlockNode", () => { expect(chain.run).toHaveBeenCalledTimes(1) }) }) + +describe("isLanguageDetected", () => { + it.for([ + { name: "detects a block without a language", input: {}, expected: true }, + { + name: "detects a block with detection on", + input: { language: "go", auto: true }, + expected: true, + }, + { + name: "leaves a block whose language was picked", + input: { language: "go" }, + expected: false, + }, + ])("$name", ({ input, expected }, { expect }) => { + expect(isLanguageDetected(detectedCode(input))).toBe(expected) + }) +}) + +function detectedCode(attrs: Record): PMNode { + return detectionSchema.nodes[CODE_BLOCK_NAME].create(attrs) +} diff --git a/web/app/components/editor/blocks/code-block/index.ts b/web/app/components/editor/blocks/code-block/index.ts index 4b6a994b..a7def1be 100644 --- a/web/app/components/editor/blocks/code-block/index.ts +++ b/web/app/components/editor/blocks/code-block/index.ts @@ -14,6 +14,7 @@ import CodeTitle from "./CodeTitle.vue" import { extendedCodeBlockLanguages } from "./languages" import { createLowlight } from "lowlight" import { Selection, TextSelection } from "prosemirror-state" +import type { Node as PMNode } from "@tiptap/pm/model" import { KeywordColor } from "./keyword" import { deleteNode } from "../../tiptap-utils/node" import { @@ -141,6 +142,22 @@ export const CodeBlock = CodeBlockLowlight.extend({ type: "document", } }, + addAttributes() { + return { + ...this.parent?.(), + auto: { + default: false, + parseHTML: (element) => element.getAttribute("data-auto") === "true", + renderHTML: (attrs) => { + if (!attrs.auto) { + return {} + } + + return { "data-auto": "true" } + }, + }, + } + }, addInputRules() { return [] }, @@ -330,3 +347,9 @@ export function setUpCodeBlockNode( }) .run() } + +// a block is detected until someone picks its language. A block without a +// language is detected too, because older blocks carry no auto attribute. +export function isLanguageDetected(node: PMNode): boolean { + return Boolean(node.attrs.auto) || !node.attrs.language +} diff --git a/web/app/components/editor/diff/change-count.ts b/web/app/components/editor/diff/change-count.ts index 6b4e477c..cde1c36a 100644 --- a/web/app/components/editor/diff/change-count.ts +++ b/web/app/components/editor/diff/change-count.ts @@ -10,6 +10,7 @@ import { } from "~/components/editor/attribute-names" import { SIMULATION_ACTIVE_ATTR } from "~/components/editor/blocks/metrics/simulation" import { + CODE_BLOCK_NAME, FILE_BLOCK_NAME, IMAGE_BLOCK_NAME, METRIC_BLOCK_NAME, @@ -45,6 +46,10 @@ const UNTRACKED_ATTRIBUTES: Record = { // attributes that change together and count as one change, per node // type. Each attribute maps to the name of its group. const ATTRIBUTE_GROUPS: Record> = { + [CODE_BLOCK_NAME]: { + language: "language", + auto: "language", + }, [FILE_BLOCK_NAME]: { src: "file", name: "file",