mirror of
https://github.com/oxynote/oxynote.git
synced 2026-10-17 02:49:18 +02:00
improve codeblock lang autodetection
This commit is contained in:
swithek
parent
a6efcdf17d
commit
4df77fdadd
|
|
@ -73,6 +73,24 @@ export const CodeBlock = CodeBlockLowlight.extend<CodeBlockOptions>({
|
|||
],
|
||||
]
|
||||
},
|
||||
addAttributes() {
|
||||
return {
|
||||
...this.parent?.(),
|
||||
auto: {
|
||||
default: false,
|
||||
parseHTML: (element) =>
|
||||
element.getAttribute("data-auto") ===
|
||||
"true",
|
||||
renderHTML: (attrs) => {
|
||||
if (!attrs.auto) {
|
||||
return {}
|
||||
}
|
||||
|
||||
return { "data-auto": "true" }
|
||||
},
|
||||
},
|
||||
}
|
||||
},
|
||||
addOptions(): CodeBlockOptions {
|
||||
const parent = this.parent ? this.parent() : null
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -2,7 +2,9 @@ import { flushPromises, type VueWrapper } from "@vue/test-utils"
|
|||
import { beforeEach, describe, it, vi } from "vitest"
|
||||
import CodeButtons from "./CodeButtons.vue"
|
||||
import {
|
||||
commandArgs,
|
||||
makeEditor,
|
||||
makeEditorState,
|
||||
makeNode,
|
||||
mountNodeView,
|
||||
type EditorStub,
|
||||
|
|
@ -14,6 +16,7 @@ const GO_SNIPPET =
|
|||
'package main\n\nimport "fmt"\n\nfunc main() {\n\tfmt.Println("hi")\n}\n'
|
||||
|
||||
const COPIED_RESET_MS = 700
|
||||
const STORE_DELAY_MS = 500
|
||||
|
||||
function mountButtons(
|
||||
options: {
|
||||
|
|
@ -110,7 +113,27 @@ describe("<CodeButtons>", { concurrent: false }, () => {
|
|||
emitFrom(wrapper, Select, "update:modelValue", "rust")
|
||||
|
||||
expect(updateAttributes).toHaveBeenCalledTimes(1)
|
||||
expect(updateAttributes).toHaveBeenCalledWith({ language: "rust" })
|
||||
expect(updateAttributes).toHaveBeenCalledWith({
|
||||
language: "rust",
|
||||
auto: false,
|
||||
})
|
||||
})
|
||||
|
||||
it("turns detection off when the user picks the detected language", async ({
|
||||
expect,
|
||||
}) => {
|
||||
const updateAttributes = vi.fn()
|
||||
const wrapper = await mountButtons({
|
||||
attrs: { language: "rust", auto: true },
|
||||
updateAttributes: updateAttributes,
|
||||
})
|
||||
|
||||
emitFrom(wrapper, Select, "update:modelValue", "rust")
|
||||
|
||||
expect(updateAttributes).toHaveBeenCalledWith({
|
||||
language: "rust",
|
||||
auto: false,
|
||||
})
|
||||
})
|
||||
|
||||
it("stores nothing when the picked language is already set", async ({
|
||||
|
|
@ -142,9 +165,7 @@ describe("<CodeButtons>", { concurrent: false }, () => {
|
|||
),
|
||||
})
|
||||
|
||||
editor.on.mock.calls.forEach(([, handler]) => {
|
||||
;(handler as () => void)()
|
||||
})
|
||||
reportUpdate(editor)
|
||||
await nextTick()
|
||||
|
||||
expect(editor.on).toHaveBeenCalledTimes(1)
|
||||
|
|
@ -161,14 +182,95 @@ describe("<CodeButtons>", { concurrent: false }, () => {
|
|||
editor: editor,
|
||||
})
|
||||
|
||||
editor.on.mock.calls.forEach(([, handler]) => {
|
||||
;(handler as () => void)()
|
||||
})
|
||||
reportUpdate(editor)
|
||||
await nextTick()
|
||||
|
||||
expect(selectedLanguage(wrapper)).toBe("Python")
|
||||
})
|
||||
|
||||
it("stores the detected language once the user stops typing", async ({
|
||||
expect,
|
||||
}) => {
|
||||
const editor = makeEditor()
|
||||
await mountEdited(editor)
|
||||
|
||||
reportUpdate(editor)
|
||||
await vi.advanceTimersByTimeAsync(STORE_DELAY_MS - 1)
|
||||
|
||||
expect(commandArgs(editor.commands, "command")).toBeUndefined()
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1)
|
||||
|
||||
const tr = { setNodeAttribute: vi.fn(), setMeta: vi.fn() }
|
||||
const [store] = commandArgs(editor.commands, "command") as [
|
||||
(props: { tr: unknown }) => boolean,
|
||||
]
|
||||
store({ tr: tr })
|
||||
|
||||
expect(tr.setNodeAttribute.mock.calls).toEqual([
|
||||
[0, "language", "go"],
|
||||
[0, "auto", true],
|
||||
])
|
||||
expect(tr.setMeta).toHaveBeenCalledWith("addToHistory", false)
|
||||
})
|
||||
|
||||
it("stores nothing for a remote change", async ({ expect }) => {
|
||||
const editor = makeEditor()
|
||||
await mountEdited(editor)
|
||||
|
||||
reportUpdate(editor, true)
|
||||
await vi.advanceTimersByTimeAsync(STORE_DELAY_MS)
|
||||
|
||||
expect(commandArgs(editor.commands, "command")).toBeUndefined()
|
||||
})
|
||||
|
||||
it("stores nothing while the caret is outside the block", async ({
|
||||
expect,
|
||||
}) => {
|
||||
const editor = makeEditor({
|
||||
state: makeEditorState({ from: 10, to: 10 }),
|
||||
})
|
||||
await mountEdited(editor)
|
||||
|
||||
reportUpdate(editor)
|
||||
await vi.advanceTimersByTimeAsync(STORE_DELAY_MS)
|
||||
|
||||
expect(commandArgs(editor.commands, "command")).toBeUndefined()
|
||||
})
|
||||
|
||||
it("stores nothing over a language the user picked", async ({ expect }) => {
|
||||
const editor = makeEditor()
|
||||
await mountEdited(editor, { language: "python" })
|
||||
|
||||
reportUpdate(editor)
|
||||
await vi.advanceTimersByTimeAsync(STORE_DELAY_MS)
|
||||
|
||||
expect(commandArgs(editor.commands, "command")).toBeUndefined()
|
||||
})
|
||||
|
||||
it("stores nothing when the detected language is already stored", async ({
|
||||
expect,
|
||||
}) => {
|
||||
const editor = makeEditor()
|
||||
await mountEdited(editor, { language: "go", auto: true })
|
||||
|
||||
reportUpdate(editor)
|
||||
await vi.advanceTimersByTimeAsync(STORE_DELAY_MS)
|
||||
|
||||
expect(commandArgs(editor.commands, "command")).toBeUndefined()
|
||||
})
|
||||
|
||||
it("stores nothing once unmounted", async ({ expect }) => {
|
||||
const editor = makeEditor()
|
||||
const wrapper = await mountEdited(editor)
|
||||
|
||||
reportUpdate(editor)
|
||||
wrapper.unmount()
|
||||
await vi.advanceTimersByTimeAsync(STORE_DELAY_MS)
|
||||
|
||||
expect(commandArgs(editor.commands, "command")).toBeUndefined()
|
||||
})
|
||||
|
||||
it("stops listening for editor updates once unmounted", async ({
|
||||
expect,
|
||||
}) => {
|
||||
|
|
@ -285,3 +387,32 @@ describe("<CodeButtons>", { concurrent: false }, () => {
|
|||
)
|
||||
})
|
||||
})
|
||||
|
||||
// run the editor's update listeners as a change would. A remote change
|
||||
// carries the sync plugin's meta, a local one carries none.
|
||||
function reportUpdate(editor: EditorStub, remote = false) {
|
||||
const transaction = { getMeta: () => (remote ? {} : undefined) }
|
||||
|
||||
editor.on.mock.calls.forEach(([, handler]) => {
|
||||
;(handler as (props: { transaction: unknown }) => void)({
|
||||
transaction: transaction,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// mount an empty block, then give it the Go snippet as an edit would
|
||||
async function mountEdited(
|
||||
editor: EditorStub,
|
||||
attrs: Record<string, unknown> = {},
|
||||
) {
|
||||
const wrapper = await mountButtons({ attrs: attrs, editor: editor })
|
||||
await wrapper.setProps({
|
||||
node: makeNode(
|
||||
{ uid: "code-1", language: null, ...attrs },
|
||||
{ textContent: GO_SNIPPET },
|
||||
),
|
||||
})
|
||||
vi.useFakeTimers()
|
||||
|
||||
return wrapper
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,12 +5,20 @@ import {
|
|||
defaultExtendedCodeBlockLanguage,
|
||||
extendedCodeBlockLanguageOptions,
|
||||
} from "./languages"
|
||||
import { lowlight, type CodeBlockOptions } from "./index"
|
||||
import { isLanguageDetected, type CodeBlockOptions } from "./index"
|
||||
import { detectLanguage } from "./detection"
|
||||
import { isChangeOrigin } from "@tiptap/extension-collaboration"
|
||||
import type { Transaction } from "@tiptap/pm/state"
|
||||
|
||||
// detection runs every grammar over the code, so it waits until the code
|
||||
// stops changing
|
||||
const STORE_DELAY_MS = 500
|
||||
|
||||
const props = defineProps({
|
||||
...nodeViewProps,
|
||||
})
|
||||
const { isEditable } = useEditorMeta()
|
||||
|
||||
const isCopied = ref(false)
|
||||
const typeClass = computed(() => {
|
||||
return (props.extension.options as CodeBlockOptions).type === "comment"
|
||||
|
|
@ -20,48 +28,9 @@ const typeClass = computed(() => {
|
|||
|
||||
const detectedLanguage = ref(defaultExtendedCodeBlockLanguage)
|
||||
|
||||
function detectLanguage() {
|
||||
if (props.node.attrs.language) {
|
||||
return
|
||||
}
|
||||
|
||||
const content = props.node.textContent
|
||||
if (!content.trim()) {
|
||||
detectedLanguage.value = defaultExtendedCodeBlockLanguage
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
const result = lowlight.highlightAuto(content)
|
||||
|
||||
if (
|
||||
result.data?.language &&
|
||||
result.data.relevance &&
|
||||
result.data.relevance >= 2
|
||||
) {
|
||||
detectedLanguage.value = result.data.language
|
||||
return
|
||||
}
|
||||
} catch {
|
||||
// Ignore errors
|
||||
}
|
||||
|
||||
detectedLanguage.value = defaultExtendedCodeBlockLanguage
|
||||
}
|
||||
|
||||
onBeforeMount(() => {
|
||||
detectLanguage()
|
||||
})
|
||||
onMounted(() => {
|
||||
props.editor.on("update", detectLanguage)
|
||||
})
|
||||
onUnmounted(() => {
|
||||
props.editor.off("update", detectLanguage)
|
||||
})
|
||||
|
||||
const currentLang = computed({
|
||||
get: () => {
|
||||
// If language is null (auto mode), show detected language
|
||||
// a block the detector has not stored yet shows its own guess
|
||||
if (!props.node.attrs.language) {
|
||||
return detectedLanguage.value
|
||||
}
|
||||
|
|
@ -69,13 +38,84 @@ const currentLang = computed({
|
|||
return props.node.attrs.language as string
|
||||
},
|
||||
set: (lang: string) => {
|
||||
// User selected a language - set it (no longer auto)
|
||||
if (props.node.attrs.language !== lang) {
|
||||
props.updateAttributes({ language: lang })
|
||||
// picking a language turns detection off, even when it is the
|
||||
// detected one
|
||||
if (props.node.attrs.language !== lang || props.node.attrs.auto) {
|
||||
props.updateAttributes({ language: lang, auto: false })
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
let storeTimeout: ReturnType<typeof setTimeout> | undefined
|
||||
|
||||
onBeforeMount(() => {
|
||||
updateDetectedLanguage()
|
||||
})
|
||||
onMounted(() => {
|
||||
props.editor.on("update", onEditorUpdate)
|
||||
})
|
||||
onUnmounted(() => {
|
||||
props.editor.off("update", onEditorUpdate)
|
||||
clearTimeout(storeTimeout)
|
||||
})
|
||||
|
||||
function updateDetectedLanguage() {
|
||||
if (props.node.attrs.language) {
|
||||
return
|
||||
}
|
||||
|
||||
detectedLanguage.value = detectLanguage(props.node.textContent)
|
||||
}
|
||||
|
||||
function onEditorUpdate({ transaction }: { transaction: Transaction }) {
|
||||
updateDetectedLanguage()
|
||||
|
||||
// only the user typing in the block stores its language, because the
|
||||
// server counts every write as an edit by its user
|
||||
if (isChangeOrigin(transaction) || !isCaretInside()) {
|
||||
return
|
||||
}
|
||||
|
||||
clearTimeout(storeTimeout)
|
||||
storeTimeout = setTimeout(storeDetectedLanguage, STORE_DELAY_MS)
|
||||
}
|
||||
|
||||
function isCaretInside(): boolean {
|
||||
const pos = props.getPos()
|
||||
if (typeof pos !== "number") {
|
||||
return false
|
||||
}
|
||||
|
||||
const { from, to } = props.editor.state.selection
|
||||
|
||||
return from >= pos && to <= pos + props.node.nodeSize
|
||||
}
|
||||
|
||||
function storeDetectedLanguage() {
|
||||
const pos = props.getPos()
|
||||
if (typeof pos !== "number" || !isLanguageDetected(props.node)) {
|
||||
return
|
||||
}
|
||||
|
||||
const language = detectLanguage(props.node.textContent)
|
||||
if (props.node.attrs.language === language && props.node.attrs.auto) {
|
||||
return
|
||||
}
|
||||
|
||||
// the write stays out of the undo history. A tracked write after an
|
||||
// undo would clear what can be redone.
|
||||
props.editor
|
||||
.chain()
|
||||
.command(({ tr }) => {
|
||||
tr.setNodeAttribute(pos, "language", language)
|
||||
tr.setNodeAttribute(pos, "auto", true)
|
||||
tr.setMeta("addToHistory", false)
|
||||
|
||||
return true
|
||||
})
|
||||
.run()
|
||||
}
|
||||
|
||||
async function copyCodeBlock() {
|
||||
try {
|
||||
await navigator.clipboard.writeText(props.node.textContent || "")
|
||||
|
|
|
|||
254
web/app/components/editor/blocks/code-block/detection.test.ts
Normal file
254
web/app/components/editor/blocks/code-block/detection.test.ts
Normal file
|
|
@ -0,0 +1,254 @@
|
|||
import { describe, it } from "vitest"
|
||||
import { detectLanguage } from "./detection"
|
||||
|
||||
const SNIPPETS = [
|
||||
{
|
||||
name: 'detects bash in a snippet starting "#!/usr/bin/env bash"',
|
||||
input:
|
||||
'#!/usr/bin/env bash\nset -euo pipefail\n\nfor f in *.log; do\n echo "processing $f"\n gzip "$f"\ndone',
|
||||
expected: "bash",
|
||||
},
|
||||
{
|
||||
name: 'detects bash in a snippet starting "docker compose up -d"',
|
||||
input: "docker compose up -d\nmake e2e-dev\npnpm install",
|
||||
expected: "bash",
|
||||
},
|
||||
{
|
||||
name: 'detects c in a snippet starting "#include <stdio.h>"',
|
||||
input:
|
||||
'#include <stdio.h>\n\nint main(void) {\n printf("hello\\n");\n return 0;\n}',
|
||||
expected: "c",
|
||||
},
|
||||
{
|
||||
name: 'detects cpp in a snippet starting "#include <iostream>"',
|
||||
input:
|
||||
"#include <iostream>\n#include <vector>\n\nint main() {\n std::vector<int> v = {1, 2, 3};\n for (auto x : v) {\n std::cout << x << std::endl;\n }\n return 0;\n}",
|
||||
expected: "cpp",
|
||||
},
|
||||
{
|
||||
name: 'detects csharp in a snippet starting "using System;"',
|
||||
input:
|
||||
"using System;\nusing System.Collections.Generic;\n\nnamespace App\n{\n public class Program\n {\n public static void Main(string[] args)\n {\n var items = new List<string>();\n foreach (var item in items)\n {\n Console.WriteLine(item);\n }\n }\n }\n}",
|
||||
expected: "csharp",
|
||||
},
|
||||
{
|
||||
name: 'detects csharp in a snippet starting "public class User"',
|
||||
input:
|
||||
'public class User\n{\n public string Id { get; set; }\n public string Name { get; set; }\n\n public async Task<User> LoadAsync(HttpClient client)\n {\n var json = await client.GetStringAsync("/me");\n return JsonSerializer.Deserialize<User>(json);\n }\n}',
|
||||
expected: "csharp",
|
||||
},
|
||||
{
|
||||
name: 'detects css in a snippet starting ".button {"',
|
||||
input:
|
||||
".button {\n color: red;\n padding: 0.5rem 1rem;\n}\n\n.button:hover {\n color: blue;\n}",
|
||||
expected: "css",
|
||||
},
|
||||
{
|
||||
name: 'detects curl in a snippet starting "curl -X POST https://api.example"',
|
||||
input:
|
||||
'curl -X POST https://api.example.com/v1/items \\\n -H "Authorization: Bearer $TOKEN" \\\n -H "Content-Type: application/json" \\\n -d \'{"name": "x"}\'',
|
||||
expected: "curl",
|
||||
},
|
||||
{
|
||||
name: 'detects diff in a snippet starting "--- a/file.txt"',
|
||||
input:
|
||||
"--- a/file.txt\n+++ b/file.txt\n@@ -1,3 +1,3 @@\n-old line\n+new line\n context",
|
||||
expected: "diff",
|
||||
},
|
||||
{
|
||||
name: 'detects go in a snippet starting "package main"',
|
||||
input:
|
||||
'package main\n\nimport (\n\t"fmt"\n\t"net/http"\n)\n\ntype Server struct {\n\taddr string\n}\n\nfunc (s *Server) Run() error {\n\tmux := http.NewServeMux()\n\tmux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {\n\t\tfmt.Fprintln(w, "hi")\n\t})\n\n\tif err := http.ListenAndServe(s.addr, mux); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}',
|
||||
expected: "go",
|
||||
},
|
||||
{
|
||||
name: 'detects go in a snippet starting "func Sum(values []int) int {"',
|
||||
input:
|
||||
"func Sum(values []int) int {\n\ttotal := 0\n\tfor _, v := range values {\n\t\ttotal += v\n\t}\n\n\treturn total\n}",
|
||||
expected: "go",
|
||||
},
|
||||
{
|
||||
name: 'detects go in a snippet starting "type User struct {"',
|
||||
input:
|
||||
'type User struct {\n\tID string `json:"id"`\n\tName string `json:"name"`\n\tCreatedAt time.Time `json:"created_at"`\n}',
|
||||
expected: "go",
|
||||
},
|
||||
{
|
||||
name: 'detects graphql in a snippet starting "query GetUser($id: ID!) {"',
|
||||
input:
|
||||
"query GetUser($id: ID!) {\n user(id: $id) {\n id\n name\n posts(first: 10) {\n title\n }\n }\n}",
|
||||
expected: "graphql",
|
||||
},
|
||||
{
|
||||
name: 'detects ini in a snippet starting "[server]"',
|
||||
input:
|
||||
"[server]\nhost = 0.0.0.0\nport = 8080\n\n[database]\nurl = postgres://localhost/db",
|
||||
expected: "ini",
|
||||
},
|
||||
{
|
||||
name: 'detects java in a snippet starting "import java.util.List;"',
|
||||
input:
|
||||
'import java.util.List;\n\npublic class Main {\n public static void main(String[] args) {\n List<String> items = List.of("a", "b");\n for (String item : items) {\n System.out.println(item);\n }\n }\n}',
|
||||
expected: "java",
|
||||
},
|
||||
{
|
||||
name: 'detects javascript in a snippet starting "const express = require(\\"express"',
|
||||
input:
|
||||
'const express = require("express")\nconst app = express()\n\napp.get("/", (req, res) => {\n\tres.send("hello")\n})\n\napp.listen(3000, () => console.log("up"))',
|
||||
expected: "javascript",
|
||||
},
|
||||
{
|
||||
name: 'detects json in a snippet starting "{"',
|
||||
input:
|
||||
'{\n "name": "oxynote",\n "version": "1.0.0",\n "private": true,\n "scripts": {\n "dev": "nuxt dev",\n "build": "nuxt build"\n },\n "tags": ["a", "b"],\n "count": 3\n}',
|
||||
expected: "json",
|
||||
},
|
||||
{
|
||||
name: 'detects json in a snippet starting "{\\"id\\": 12, \\"ok\\": true}"',
|
||||
input: '{"id": 12, "ok": true}',
|
||||
expected: "json",
|
||||
},
|
||||
{
|
||||
name: 'detects json in a snippet starting "["',
|
||||
input: '[\n { "id": 1, "name": "a" },\n { "id": 2, "name": "b" }\n]',
|
||||
expected: "json",
|
||||
},
|
||||
{
|
||||
name: 'detects kotlin in a snippet starting "fun main() {"',
|
||||
input:
|
||||
'fun main() {\n val items = listOf("a", "b")\n for (item in items) {\n println(item)\n }\n}',
|
||||
expected: "kotlin",
|
||||
},
|
||||
{
|
||||
name: 'detects makefile in a snippet starting "CC = gcc"',
|
||||
input:
|
||||
"CC = gcc\nCFLAGS = -Wall -O2\n\napp: main.o util.o\n\t$(CC) $(CFLAGS) -o $@ $^\n\nclean:\n\trm -f *.o app",
|
||||
expected: "makefile",
|
||||
},
|
||||
{
|
||||
name: 'detects markdown in a snippet starting "# Title"',
|
||||
input:
|
||||
"# Title\n\nSome **bold** text and a [link](https://example.com).\n\n- one\n- two",
|
||||
expected: "markdown",
|
||||
},
|
||||
{
|
||||
name: 'detects php in a snippet starting "<?php"',
|
||||
input:
|
||||
"<?php\n\nfunction total(array $values): int {\n $sum = 0;\n foreach ($values as $v) {\n $sum += $v;\n }\n return $sum;\n}",
|
||||
expected: "php",
|
||||
},
|
||||
{
|
||||
name: 'detects plaintext in a snippet starting "sum(rate(http_requests_total{job"',
|
||||
input: 'sum(rate(http_requests_total{job="api"}[5m])) by (status)',
|
||||
expected: "plaintext",
|
||||
},
|
||||
{
|
||||
name: 'detects plaintext in a snippet starting "This is just a note about the de"',
|
||||
input:
|
||||
"This is just a note about the deployment.\nNothing here is code at all.",
|
||||
expected: "plaintext",
|
||||
},
|
||||
{
|
||||
name: 'detects plaintext in a snippet starting "Remember to rotate the keys befo"',
|
||||
input: "Remember to rotate the keys before Friday, and tell the team.",
|
||||
expected: "plaintext",
|
||||
},
|
||||
{
|
||||
name: 'detects plaintext in a snippet starting "GET /api/users/42"',
|
||||
input: "GET /api/users/42\nHost: example.com\nAccept: application/json",
|
||||
expected: "plaintext",
|
||||
},
|
||||
{
|
||||
name: 'detects python in a snippet starting "import os"',
|
||||
input:
|
||||
'import os\nfrom typing import List\n\ndef total(values: List[int]) -> int:\n result = 0\n for v in values:\n result += v\n return result\n\nif __name__ == "__main__":\n print(total([1, 2, 3]))',
|
||||
expected: "python",
|
||||
},
|
||||
{
|
||||
name: 'detects ruby in a snippet starting "class User"',
|
||||
input:
|
||||
'class User\n attr_reader :name\n\n def initialize(name)\n @name = name\n end\n\n def greet\n puts "hi #{@name}"\n end\nend',
|
||||
expected: "ruby",
|
||||
},
|
||||
{
|
||||
name: 'detects rust in a snippet starting "use std::collections::HashMap;"',
|
||||
input:
|
||||
'use std::collections::HashMap;\n\nfn main() {\n let mut map: HashMap<String, i32> = HashMap::new();\n map.insert("a".to_string(), 1);\n for (k, v) in &map {\n println!("{k}: {v}");\n }\n}',
|
||||
expected: "rust",
|
||||
},
|
||||
{
|
||||
name: 'detects shell in a snippet starting "$ docker compose up -d"',
|
||||
input: "$ docker compose up -d\n$ make e2e-dev",
|
||||
expected: "shell",
|
||||
},
|
||||
{
|
||||
name: 'detects sql in a snippet starting "SELECT u.id, u.name, count(o.id)"',
|
||||
input:
|
||||
"SELECT u.id, u.name, count(o.id) AS orders\nFROM users u\nLEFT JOIN orders o ON o.user_id = u.id\nWHERE u.active = true\nGROUP BY u.id\nORDER BY orders DESC;",
|
||||
expected: "sql",
|
||||
},
|
||||
{
|
||||
name: 'detects swift in a snippet starting "import Foundation"',
|
||||
input:
|
||||
'import Foundation\n\nstruct User {\n let name: String\n}\n\nfunc greet(_ user: User) -> String {\n return "hi \\(user.name)"\n}',
|
||||
expected: "swift",
|
||||
},
|
||||
{
|
||||
name: 'detects typescript in a snippet starting "interface User {"',
|
||||
input:
|
||||
'interface User {\n\tid: string\n\tname: string\n}\n\nexport async function loadUser(id: string): Promise<User> {\n\tconst res = await fetch(`/api/users/${id}`)\n\tif (!res.ok) {\n\t\tthrow new Error("failed")\n\t}\n\n\treturn (await res.json()) as User\n}',
|
||||
expected: "typescript",
|
||||
},
|
||||
{
|
||||
name: 'detects xml in a snippet starting "<!DOCTYPE html>"',
|
||||
input:
|
||||
'<!DOCTYPE html>\n<html>\n <head><title>x</title></head>\n <body>\n <div class="a">hi</div>\n </body>\n</html>',
|
||||
expected: "xml",
|
||||
},
|
||||
{
|
||||
name: 'detects yaml in a snippet starting "services:"',
|
||||
input:
|
||||
'services:\n web:\n image: nginx:latest\n ports:\n - "8080:80"\n environment:\n - KEY=value',
|
||||
expected: "yaml",
|
||||
},
|
||||
]
|
||||
|
||||
// the highlighter's keyword count alone names another language for this one
|
||||
const GO_WITH_SHARED_KEYWORDS =
|
||||
'var (\n\tErrNotFound = errors.New("not found")\n\tErrInvalid = errors.New("invalid")\n)\n\ntype Store interface {\n\tGet(ctx context.Context, id string) (User, error)\n\tPut(ctx context.Context, u User) error\n}'
|
||||
|
||||
const NO_CLEAR_LEAD =
|
||||
'public := os.Getenv("PUBLIC_URL")\nvar client = &http.Client{Timeout: 5 * time.Second}\nvar result string\nif public == "" {\n\tresult = "none"\n}\nnew := strings.TrimSpace(result)'
|
||||
|
||||
const WEAK_EVIDENCE =
|
||||
'sum(rate(http_requests_total{job="api"}[5m])) by (status)'
|
||||
|
||||
describe("detectLanguage", () => {
|
||||
it.for(SNIPPETS)("$name", ({ input, expected }, { expect }) => {
|
||||
expect(detectLanguage(input)).toBe(expected)
|
||||
})
|
||||
|
||||
it("detects go where the shared keywords favour another language", ({
|
||||
expect,
|
||||
}) => {
|
||||
expect(detectLanguage(GO_WITH_SHARED_KEYWORDS)).toBe("go")
|
||||
})
|
||||
|
||||
it("detects json written on one line", ({ expect }) => {
|
||||
expect(detectLanguage('{"id": 12, "ok": true}')).toBe("json")
|
||||
})
|
||||
|
||||
it("falls back to plaintext for blank code", ({ expect }) => {
|
||||
expect(detectLanguage(" \n ")).toBe("plaintext")
|
||||
})
|
||||
|
||||
it("falls back to plaintext when no language has a clear lead", ({
|
||||
expect,
|
||||
}) => {
|
||||
expect(detectLanguage(NO_CLEAR_LEAD)).toBe("plaintext")
|
||||
})
|
||||
|
||||
it("falls back to plaintext when the evidence is weak", ({ expect }) => {
|
||||
expect(detectLanguage(WEAK_EVIDENCE)).toBe("plaintext")
|
||||
})
|
||||
})
|
||||
184
web/app/components/editor/blocks/code-block/detection.ts
Normal file
184
web/app/components/editor/blocks/code-block/detection.ts
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
import { lowlight } from "./index"
|
||||
import { defaultExtendedCodeBlockLanguage } from "./languages"
|
||||
|
||||
// one matching pattern counts as much as this many keyword matches
|
||||
const PATTERN_WEIGHT = 5
|
||||
|
||||
// the best language needs this score and this lead over the next one. A
|
||||
// block that reaches neither stays plaintext.
|
||||
const MIN_SCORE = 5
|
||||
const MIN_LEAD = 3
|
||||
|
||||
// constructs only one of the languages writes. The highlighter scores a
|
||||
// language by its keywords alone, and many languages share those.
|
||||
const LANGUAGE_PATTERNS: Record<string, RegExp[]> = {
|
||||
bash: [
|
||||
/^#!.*\b(ba|z)?sh\b/,
|
||||
/^\s*(sudo |cd |export \w+=|echo |source )/m,
|
||||
/^\s*(git|docker|kubectl|helm|npm|pnpm|yarn|npx|make|brew|apt(-get)?|pip3?|cargo|go|terraform|aws|gcloud|ssh|scp|wget|tar|mkdir|chmod|cat|grep|ls|rm|cp|mv) [\w.\-/]/m,
|
||||
/\b(then|fi|done|esac)\s*$/m,
|
||||
/\$\{?\w+\}?"|\$\(/,
|
||||
],
|
||||
c: [
|
||||
/^#include <\w+\.h>/m,
|
||||
/\b(printf|malloc|sizeof)\(/,
|
||||
/^(int|void|char) \w+\([^)]*\)\s*\{/m,
|
||||
],
|
||||
cpp: [
|
||||
/^#include <[a-z_]+>/m,
|
||||
/\bstd::/,
|
||||
/^(template|namespace|class) .*[<{]/m,
|
||||
/\b(cout|cin|endl)\b/,
|
||||
],
|
||||
csharp: [
|
||||
/^using [\w.]+;/m,
|
||||
/^namespace [\w.]+\s*[{;]?$/m,
|
||||
/\{ get; (set|init); \}/,
|
||||
/\b(Console\.Write|string\[\] args|async Task|foreach \(|IEnumerable<)/,
|
||||
/\.(Where|Select|OrderBy|ToList)\(/,
|
||||
/\b(Task<|IActionResult|public record |private readonly )/,
|
||||
/^\s*\[(Http\w+|ApiController|Route|Fact|Test)\b/m,
|
||||
],
|
||||
css: [
|
||||
/^[.#][\w-]+[^\n{};=(]*\{\s*$/m,
|
||||
/^(html|body|a|p|div|span|h[1-6]|ul|li|table|input|button|img)\b[^\n{};=(]*\{\s*$/m,
|
||||
/^\s*[a-z-]+:\s*[^;{}\n]+;\s*$/m,
|
||||
/(^|\s)(@media|@import|:root|!important)\b/m,
|
||||
],
|
||||
curl: [/^\s*curl\s/],
|
||||
diff: [
|
||||
/^diff --git /m,
|
||||
/^@@ -\d+(,\d+)? \+\d+(,\d+)? @@/m,
|
||||
/^--- .*\n\+\+\+ /m,
|
||||
],
|
||||
go: [
|
||||
/^package \w+\s*$/m,
|
||||
/^func (\(\w+ \*?[\w.]+\) )?\w+(\[[^\]]+\])?\(/m,
|
||||
/\w+(, \w+)* := /,
|
||||
/^type \w+ (struct|interface) \{/m,
|
||||
/\berr [!=]= nil\b|\bfunc\(|\bdefer |\bchan |\brange /,
|
||||
/^import \(\s*$|^import "[\w./-]+"$/m,
|
||||
],
|
||||
graphql: [
|
||||
/^\s*(query|mutation|subscription|fragment)\b[^=;\n]*\{\s*$/m,
|
||||
/^\s*\w+(\([^)]*\))?:\s*\[?\w+!?\]?!?\s*$/m,
|
||||
/\$\w+:\s*\[?\w+!?\]?/,
|
||||
],
|
||||
ini: [/^\[[\w.\- ]+\]\s*$/m, /^[\w.-]+ = \S/m],
|
||||
java: [
|
||||
/^import (static )?(java|javax|org|com)\.[\w.*]+;/m,
|
||||
/\bSystem\.(out|err)\.|String\[\] args/,
|
||||
/^\s*@(Override|Autowired|RestController|GetMapping|PostMapping|Test|Entity)\b/m,
|
||||
/\b(public|private|protected) (static )?(final )?[\w<>[\]]+ \w+\([^)]*\)\s*(throws \w+\s*)?\{/,
|
||||
],
|
||||
javascript: [
|
||||
/\b(const|let) \w+ = /,
|
||||
/=>\s*[{(\w]/,
|
||||
/\b(require\(|console\.\w+\(|document\.|module\.exports|window\.)/,
|
||||
/\b(async )?function\*? \w*\(/,
|
||||
/^(import|export) .* from ["']/m,
|
||||
],
|
||||
json: [
|
||||
/^\s*\{\s*("[^"\n]*"\s*:|\}\s*$)/,
|
||||
/^\s*\[\s*([{["\d\]-]|true|false|null)/,
|
||||
/^\s*"[^"\n]*"\s*:\s*("|\d|true|false|null|\{|\[)/m,
|
||||
],
|
||||
kotlin: [
|
||||
/\bfun \w+\(/,
|
||||
/\b(val|var) \w+(: \w+)? = /,
|
||||
/\b(println\(|listOf\(|data class |companion object)/,
|
||||
],
|
||||
makefile: [/^\.PHONY:/m, /^[\w.\-/%]+:[^=\n]*\n\t\S/m, /\$\((\w+)\)|\$@|\$</],
|
||||
markdown: [
|
||||
/^#{1,6} \S.*\n\s*\n/m,
|
||||
/\[[^\]\n]+\]\([^)\n]+\)/,
|
||||
/(\*\*|__)[^*_\n]+(\*\*|__)/,
|
||||
/^```/m,
|
||||
],
|
||||
php: [/^\s*<\?php/, /\$\w+->\w+|\$this\b/, /\bfunction \w+\([^)]*\$\w+/],
|
||||
python: [
|
||||
/^\s*(def|class) \w+[^\n{;]*:\s*(#.*)?$/m,
|
||||
/^from [\w.]+ import [\w*, ]+$/m,
|
||||
/^import \w+(\.\w+)*( as \w+)?$/m,
|
||||
/\b(self\.\w+|__\w+__|print\(|elif |None\b|True\b|False\b)/,
|
||||
/^\s*(if|for|while|with|try|except)\b[^\n{;]*:\s*$/m,
|
||||
],
|
||||
ruby: [
|
||||
/^\s*def \w+[^\n:{]*$/m,
|
||||
/^\s*end\s*$/m,
|
||||
/\b(attr_(reader|writer|accessor)|puts |require ')/,
|
||||
/\bdo \|[\w, ]+\|/,
|
||||
/@\w+ = /,
|
||||
],
|
||||
rust: [
|
||||
/^use \w+(::[\w{}*, ]+)+;/m,
|
||||
/\bfn \w+(<[^>]+>)?\(/,
|
||||
/\blet (mut )?\w+/,
|
||||
/^\s*#\[\w+/m,
|
||||
/\b(impl|pub (fn|struct|enum)|println!|Some\(|Ok\(|&str\b)/,
|
||||
],
|
||||
shell: [/^\$ \S/m],
|
||||
sql: [
|
||||
/^\s*(select)\b[\s\S]*\bfrom\b/im,
|
||||
/^\s*(insert into|delete from|create (table|index|view)|alter table|drop table)\b/im,
|
||||
/^\s*update \w+ set\b/im,
|
||||
/\b(where|group by|order by|left join|inner join|primary key|not null)\b/i,
|
||||
],
|
||||
swift: [
|
||||
/^import (Foundation|UIKit|SwiftUI)\b/m,
|
||||
/\bfunc \w+\([^)]*\) -> /,
|
||||
/\b(guard let|if let|let \w+: \w+)/,
|
||||
/\\\(\w+/,
|
||||
],
|
||||
typescript: [
|
||||
/\b(interface|type) \w+(<[^>]+>)? (=|\{|extends)/,
|
||||
/[\w)\]]\??: (string|number|boolean|unknown|void|any|Promise<|Record<|\w+\[\])/,
|
||||
/\b(as (const|unknown|string|number|\w+)\b|import type |readonly |enum \w+ \{)/,
|
||||
/\bexport (default |async )?(function|const|class|interface|type)\b/,
|
||||
],
|
||||
xml: [
|
||||
/^\s*<(!DOCTYPE|\?xml|[a-zA-Z][\w:-]*)(\s[^<>]*)?\/?>/,
|
||||
/<\/[a-zA-Z][\w:-]*>\s*$/m,
|
||||
/<[a-zA-Z][\w:-]*(\s+[\w:-]+="[^"]*")+\s*\/?>/,
|
||||
],
|
||||
yaml: [/^[\w.-]+:\s*$/m, /^\s+- [\w"'{[]/m, /^\s+[\w.-]+: \S/m, /^---\s*$/m],
|
||||
}
|
||||
|
||||
export function detectLanguage(code: string): string {
|
||||
if (!code.trim()) {
|
||||
return defaultExtendedCodeBlockLanguage
|
||||
}
|
||||
|
||||
let bestLanguage = defaultExtendedCodeBlockLanguage
|
||||
let bestScore = 0
|
||||
let nextScore = 0
|
||||
|
||||
for (const [language, patterns] of Object.entries(LANGUAGE_PATTERNS)) {
|
||||
const score = languageScore(language, patterns, code)
|
||||
|
||||
if (score > bestScore) {
|
||||
nextScore = bestScore
|
||||
bestScore = score
|
||||
bestLanguage = language
|
||||
} else if (score > nextScore) {
|
||||
nextScore = score
|
||||
}
|
||||
}
|
||||
|
||||
if (bestScore < MIN_SCORE || bestScore - nextScore < MIN_LEAD) {
|
||||
return defaultExtendedCodeBlockLanguage
|
||||
}
|
||||
|
||||
return bestLanguage
|
||||
}
|
||||
|
||||
function languageScore(
|
||||
language: string,
|
||||
patterns: RegExp[],
|
||||
code: string,
|
||||
): number {
|
||||
const relevance = lowlight.highlight(language, code).data?.relevance ?? 0
|
||||
const matches = patterns.filter((pattern) => pattern.test(code)).length
|
||||
|
||||
return relevance + matches * PATTERN_WEIGHT
|
||||
}
|
||||
|
|
@ -12,6 +12,7 @@ import type { CodeBlockOptions } from "./index"
|
|||
import {
|
||||
CodeBlock,
|
||||
CodeBlockTitle,
|
||||
isLanguageDetected,
|
||||
lowlight,
|
||||
setUpCodeBlockNode,
|
||||
TitledCodeBlock,
|
||||
|
|
@ -106,6 +107,17 @@ function rightSide(...children: PMNode[]): PMNode {
|
|||
|
||||
const docOf = docBuilder(schema)
|
||||
|
||||
const detectionSchema = new Schema({
|
||||
nodes: {
|
||||
doc: { content: `${CODE_BLOCK_NAME}+` },
|
||||
[CODE_BLOCK_NAME]: {
|
||||
content: "text*",
|
||||
attrs: { language: { default: null }, auto: { default: false } },
|
||||
},
|
||||
text: {},
|
||||
},
|
||||
})
|
||||
|
||||
describe("lowlight", () => {
|
||||
it("registers curl alongside the common grammars", ({ expect }) => {
|
||||
expect(lowlight.listLanguages()).toContain("curl")
|
||||
|
|
@ -393,6 +405,22 @@ describe("CodeBlock", () => {
|
|||
})
|
||||
})
|
||||
|
||||
describe("addAttributes", () => {
|
||||
it("leaves detection off for a block that does not name it", ({
|
||||
expect,
|
||||
}) => {
|
||||
const addAttributes = getExtensionField<
|
||||
() => Record<string, { default: unknown }>
|
||||
>(
|
||||
CodeBlock,
|
||||
"addAttributes",
|
||||
extensionContext(CodeBlock, CODE_BLOCK_NAME),
|
||||
)
|
||||
|
||||
expect(addAttributes().auto?.default).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe("addKeyboardShortcuts", () => {
|
||||
it("keeps the inherited shortcuts alongside its own", ({ expect }) => {
|
||||
const run = shortcutsAt(CodeBlock, CODE_BLOCK_NAME, docOf(code("ab")), 1)
|
||||
|
|
@ -731,3 +759,25 @@ describe("setUpCodeBlockNode", () => {
|
|||
expect(chain.run).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe("isLanguageDetected", () => {
|
||||
it.for([
|
||||
{ name: "detects a block without a language", input: {}, expected: true },
|
||||
{
|
||||
name: "detects a block with detection on",
|
||||
input: { language: "go", auto: true },
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "leaves a block whose language was picked",
|
||||
input: { language: "go" },
|
||||
expected: false,
|
||||
},
|
||||
])("$name", ({ input, expected }, { expect }) => {
|
||||
expect(isLanguageDetected(detectedCode(input))).toBe(expected)
|
||||
})
|
||||
})
|
||||
|
||||
function detectedCode(attrs: Record<string, unknown>): PMNode {
|
||||
return detectionSchema.nodes[CODE_BLOCK_NAME].create(attrs)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import CodeTitle from "./CodeTitle.vue"
|
|||
import { extendedCodeBlockLanguages } from "./languages"
|
||||
import { createLowlight } from "lowlight"
|
||||
import { Selection, TextSelection } from "prosemirror-state"
|
||||
import type { Node as PMNode } from "@tiptap/pm/model"
|
||||
import { KeywordColor } from "./keyword"
|
||||
import { deleteNode } from "../../tiptap-utils/node"
|
||||
import {
|
||||
|
|
@ -141,6 +142,22 @@ export const CodeBlock = CodeBlockLowlight.extend<CodeBlockOptions>({
|
|||
type: "document",
|
||||
}
|
||||
},
|
||||
addAttributes() {
|
||||
return {
|
||||
...this.parent?.(),
|
||||
auto: {
|
||||
default: false,
|
||||
parseHTML: (element) => element.getAttribute("data-auto") === "true",
|
||||
renderHTML: (attrs) => {
|
||||
if (!attrs.auto) {
|
||||
return {}
|
||||
}
|
||||
|
||||
return { "data-auto": "true" }
|
||||
},
|
||||
},
|
||||
}
|
||||
},
|
||||
addInputRules() {
|
||||
return []
|
||||
},
|
||||
|
|
@ -330,3 +347,9 @@ export function setUpCodeBlockNode(
|
|||
})
|
||||
.run()
|
||||
}
|
||||
|
||||
// a block is detected until someone picks its language. A block without a
|
||||
// language is detected too, because older blocks carry no auto attribute.
|
||||
export function isLanguageDetected(node: PMNode): boolean {
|
||||
return Boolean(node.attrs.auto) || !node.attrs.language
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import {
|
|||
} from "~/components/editor/attribute-names"
|
||||
import { SIMULATION_ACTIVE_ATTR } from "~/components/editor/blocks/metrics/simulation"
|
||||
import {
|
||||
CODE_BLOCK_NAME,
|
||||
FILE_BLOCK_NAME,
|
||||
IMAGE_BLOCK_NAME,
|
||||
METRIC_BLOCK_NAME,
|
||||
|
|
@ -45,6 +46,10 @@ const UNTRACKED_ATTRIBUTES: Record<string, string[]> = {
|
|||
// attributes that change together and count as one change, per node
|
||||
// type. Each attribute maps to the name of its group.
|
||||
const ATTRIBUTE_GROUPS: Record<string, Record<string, string>> = {
|
||||
[CODE_BLOCK_NAME]: {
|
||||
language: "language",
|
||||
auto: "language",
|
||||
},
|
||||
[FILE_BLOCK_NAME]: {
|
||||
src: "file",
|
||||
name: "file",
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user