add: sacrebleu benchmark

README.md

@@ -48,4 +48,13 @@ print(results)
 ## Benchmark
 
 ### Simple
+We use the demo data shown in quick start to do this simple benchmark.
+You can check the [benchmark/simple](./benchmark/simple) for the benchmark source code.
+
+[//]: # (https://app.warp.dev/block/Mt8BOS3rllMuryMkcI4Gr5)
 ![img.png](asset/benchmark/simple.png)
+
+
+Note that bleuscore gets same result with huggingface evaluate, but sacrebleu gets different result.
+(The reason maybe related to the implementation details in sacrebleu)
+

benchmark/bench.sh

@@ -1 +1,4 @@
-hyperfine --warmup 5 --runs 100 "python simple/rs_bleuscore.py" "python simple/hf_evaluate.py"
+hyperfine --warmup 5 --runs 100   \
+  "python simple/rs_bleuscore.py" \
+  "python simple/sacre_bleu.py"   \
+  "python simple/hf_evaluate.py"

benchmark/simple/sacre_bleu.py

@@ -0,0 +1,11 @@
+from sacrebleu.metrics import BLEU
+
+
+predictions = ["hello there general kenobi", "foo bar foobar"]
+references = [
+    ["hello there general kenobi", "hello there !"],
+    ["foo bar foobar"]
+]
+
+bleu = BLEU(smooth_method="none", max_ngram_order=4, tokenize='13a')
+results = bleu.corpus_score(predictions, references)

pyproject.toml

@@ -37,7 +37,7 @@ Homepage = 'https://github.com/shenxiangzhuang/bleuscore'
 Source = 'https://github.com/shenxiangzhuang/bleuscore'
 
 [project.optional-dependencies]
-test = ["pytest", "pytest-sugar", "hypothesis", "evaluate"]
+test = ["pytest", "pytest-sugar", "hypothesis", "evaluate", "sacrebleu"]
 lint = ["black", "ruff~=0.3.7"]
 #docs = []
 #dev = []

src/bleu.rs

@@ -14,7 +14,8 @@ pub struct BleuScore {
     pub reference_length: usize,
 }
 
-/// compute the BLEU score with `Tokenizer13a` as default tokenizer
+/// compute the BLEU score with `Tokenizer13a` as default tokenizer.
+/// The implementation is based on [huggingface/nmt](https://github.com/huggingface/evaluate/blob/main/metrics/bleu/bleu.py)
 pub fn compute_score(
     references: Vec<Vec<String>>,
     predictions: Vec<String>,