Add tests

tadashi-aikawa · Aug 25, 2024 · 8f163f0 · 8f163f0
1 parent a4c4e79
commit 8f163f0
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/src/tokenizer/tokenizers/DefaultTokenizer.test.ts b/src/tokenizer/tokenizers/DefaultTokenizer.test.ts
@@ -1,6 +1,6 @@
 import { describe, expect, test } from "@jest/globals";
-import { DefaultTokenizer } from "./DefaultTokenizer";
 import type { FactoryArgs } from "../tokenizer";
+import { DefaultTokenizer } from "./DefaultTokenizer";
 
 describe.each<{
   constructorArgs: FactoryArgs;
@@ -21,8 +21,11 @@ describe.each<{
   ${{}}                                    | ${"**bold** *italic*"}   | ${["bold", "italic"]}
   ${{}}                                    | ${"__a _b __c__ d_ e__"} | ${["a", "b", "c", "d", "e"]}
   ${{ treatUnderscoreAsPartOfWord: true }} | ${"__a _b __c__ d_ e__"} | ${["__a", "_b", "__c__", "d_", "e__"]}
+  ${{}}                                    | ${"--a -b --c-- d- e--"} | ${["--a", "-b", "--c--", "d-", "e--"]}
+  ${{ treatUnderscoreAsPartOfWord: true }} | ${"--a _b -_c_- d- e__"} | ${["--a", "_b", "-_c_-", "d-", "e__"]}
   ${{}}                                    | ${"let hoge_huga = 1"}   | ${["let", "hoge", "huga", "1"]}
   ${{ treatUnderscoreAsPartOfWord: true }} | ${"let hoge_huga = 1"}   | ${["let", "hoge_huga", "1"]}
+  ${{}}                                    | ${"let hoge-huga = 1"}   | ${["let", "hoge-huga", "1"]}
   ${{}}                                    | ${'"aaa" "bbb"'}         | ${["aaa", "bbb"]}
   ${{}}                                    | ${"'aaa' 'bbb'"}         | ${["aaa", "bbb"]}
   ${{}}                                    | ${"„aaa“ „bbb“"}         | ${["aaa", "bbb"]}