Skip to content

Commit

Permalink
apply review comments:
Browse files Browse the repository at this point in the history
 - make md_eval_22 python function private
 - remove intermediate dict
 - change RTTM default
  • Loading branch information
boeddeker committed Dec 6, 2023
1 parent c564419 commit d1fd6ed
Show file tree
Hide file tree
Showing 5 changed files with 24 additions and 10 deletions.
4 changes: 2 additions & 2 deletions meeteval/der/__main__.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@

def cli():
from meeteval.wer.__main__ import CLI
from meeteval.der.md_eval import md_eval_22
from meeteval.der.md_eval import _md_eval_22

cli = CLI()

cli.add_command(md_eval_22)
cli.add_command(_md_eval_22, 'md_eval_22')

cli.run()

Expand Down
10 changes: 5 additions & 5 deletions meeteval/der/md_eval.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@

def _fix_channel(r):
return meeteval.io.rttm.RTTM([
line.replace(channel='1') # Thilo puts there usually some random value, e.g. <NA> for hyp and 0 for ref, while common default is 1
line.replace(channel='1')
# Thilo puts there usually some random value, e.g. <NA> for hyp and 0
# for ref, while dscore enforces the default to be 1
for line in r
])

Expand Down Expand Up @@ -67,7 +69,7 @@ def __add__(self, other: 'DiaErrorRate'):
)


def md_eval_22(
def _md_eval_22(
reference,
hypothesis,
average_out='{parent}/{stem}_md_eval_22.json',
Expand Down Expand Up @@ -134,15 +136,13 @@ def get_details(r, h, key, tmpdir):
def convert(string):
return decimal.Decimal(string)

details = dict(
return DiaErrorRate(
scored_speaker_time=convert(length),
# errors=float(errors),
missed_speaker_time=convert(deletions),
falarm_speaker_time=convert(insertions),
speaker_error_time=convert(substitutions),
error_rate=convert(error_rate) / 100,
)
return DiaErrorRate(**details)

per_reco = {}
with tempfile.TemporaryDirectory() as tmpdir:
Expand Down
10 changes: 9 additions & 1 deletion meeteval/io/rttm.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,18 @@ class RTTMLine(BaseLine):
SPEAKER CMU_20020319-1400_d01_NONE 1 130.430000 2.350 <NA> <NA> juliet <NA> <NA>
SPEAKER CMU_20020319-1400_d01_NONE 1 157.610000 3.060 <NA> <NA> tbc <NA> <NA>
SPEAKER CMU_20020319-1400_d01_NONE 1 130.490000 0.450 <NA> <NA> chek <NA> <NA>
Note:
The RTTM definition (Appendix A in "The 2009 (RT-09) Rich Transcription
Meeting Recognition Evaluation Plan") doesn't say anything about the
channel format or defaults, but dscore enforces a "1" for the channel
(https://github.com/nryant/dscore#rttm),
Hence, the default here is 1 for channel.
"""
type: str = 'SPEAKER'
filename: str = '<NA>'
channel: str = '<NA>'
channel: str = '1'
begin_time: 'float | int | str | decimal.Decimal' = 0
duration: 'float | int | str | decimal.Decimal' = 0
othography: 'str' = '<NA>'
Expand Down
6 changes: 4 additions & 2 deletions meeteval/wer/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -478,9 +478,11 @@ def add_argument(self, command_parser, name, p):
else:
raise AssertionError("Error in command definition", name)

def add_command(self, fn):
def add_command(self, fn, command_name=None):
if command_name is None:
command_name = fn.__name__
command_parser = self.commands.add_parser(
fn.__name__,
command_name,
add_help=False,
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
help=fn.__doc__,
Expand Down
4 changes: 4 additions & 0 deletions tests/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,11 @@ def test_burn_tcp():

def test_burn_md_eval_22():
run(f'python -m meeteval.der md_eval_22 -h hyp.stm -r ref.stm')
run(f'meeteval-der md_eval_22 -h hyp.stm -r ref.stm')
run(f'python -m meeteval.der md_eval_22 -h hyp.stm -r ref.stm --collar 0.25')
# examples for collar:
# 0: CHiME-6
# 0.25: CHiME-7 DASR


def test_burn_merge():
Expand Down

0 comments on commit d1fd6ed

Please sign in to comment.