feat: auto detect encoding for text preview (#204)

package.json

@@ -67,6 +67,7 @@
     "artplayer-plugin-danmuku": "^5.0.1",
     "asciinema-player": "^3.6.3",
     "axios": "^1.0.0",
+    "chardet": "^2.0.0",
     "clsx": "^2.0.0",
     "copy-to-clipboard": "^3.3.2",
     "crypto-js": "^4.2.0",

src/components/EncodingSelect.tsx

@@ -1,9 +1,12 @@
 import { Box } from "@hope-ui/solid"
 import { SelectWrapper } from "./Base"
+import chardet from "chardet"
+import { createEffect } from "solid-js"
 
 export function EncodingSelect(props: {
   encoding: string
   setEncoding: (encoding: string) => void
+  referenceText?: string | ArrayBuffer
 }) {
   const encodingLabels = [
     "utf-8",
@@ -48,6 +51,25 @@ export function EncodingSelect(props: {
     "x-user-defined",
     "iso-2022-cn",
   ]
+
+  createEffect(() => {
+    if (props.referenceText) {
+      let buffer: Uint8Array
+      if (typeof props.referenceText === "string") {
+        buffer = new TextEncoder().encode(props.referenceText)
+      } else {
+        buffer = new Uint8Array(props.referenceText)
+      }
+      for (let encoding of chardet.analyse(buffer)) {
+        const encodingLabel = encoding.name.toLowerCase()
+        if (encodingLabels.includes(encodingLabel)) {
+          props.setEncoding(encodingLabel)
+          return
+        }
+      }
+    }
+  })
+
   return (
     <Box
       pos="absolute"

src/components/Markdown.tsx

@@ -237,7 +237,11 @@ export function Markdown(props: {
         />
       </Show>
       <Show when={!isString}>
-        <EncodingSelect encoding={encoding()} setEncoding={setEncoding} />
+        <EncodingSelect
+          encoding={encoding()}
+          setEncoding={setEncoding}
+          referenceText={props.children}
+        />
       </Show>
       <MarkdownToc disabled={!props.toc} markdownRef={markdownRef()!} />
     </Box>

src/pages/home/previews/html.tsx

@@ -16,7 +16,11 @@ function Html(props: { children?: string | ArrayBuffer }) {
         srcdoc={text(encoding())}
       />
       <Show when={!isString}>
-        <EncodingSelect encoding={encoding()} setEncoding={setEncoding} />
+        <EncodingSelect
+          encoding={encoding()}
+          setEncoding={setEncoding}
+          referenceText={props.children}
+        />
       </Show>
     </BoxWithFullScreen>
   )

src/pages/home/previews/text-editor.tsx

@@ -45,7 +45,11 @@ function Editor(props: { data?: string | ArrayBuffer; contentType?: string }) {
   return (
     <VStack w="$full" alignItems="start" spacing="$2" pos="relative">
       <Show when={!isString}>
-        <EncodingSelect encoding={encoding()} setEncoding={setEncoding} />
+        <EncodingSelect
+          encoding={encoding()}
+          setEncoding={setEncoding}
+          referenceText={props.data}
+        />
       </Show>
       <MonacoEditorLoader
         value={text(encoding())}