Skip to content

Commit

Permalink
linting
Browse files Browse the repository at this point in the history
  • Loading branch information
sidnarayanan committed Nov 15, 2024
1 parent 360ed91 commit 56f50b5
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 10 deletions.
6 changes: 3 additions & 3 deletions tests/test_litqa.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ class TestLitQAEvaluation:
def _assert_prompt_is_valid(
qa_prompt: str, question: str, ideal: str, distractors: list[str]
):
for substr in [question, "Insufficient information", ideal, *distractors]:
for substr in (question, "Insufficient information", ideal, *distractors):
assert qa_prompt.count(substr) == 1

@pytest.mark.asyncio
Expand All @@ -25,14 +25,14 @@ async def test_from_question(self) -> None:
)
self._assert_prompt_is_valid(qa_prompt, question, ideal, distractors)

for answer, expected in [
for answer, expected in (
("the answer is 94107", LitQAEvaluation.CORRECT),
# NOTE: The below case fails this test, because the LM doesn't accept an answer not in the options.
# See https://github.com/Future-House/paper-qa/issues/693
# ("the answer is 14004", LitQAEvaluation.INCORRECT),
("the answer is 94106", LitQAEvaluation.INCORRECT),
("Insufficient information to answer", LitQAEvaluation.UNSURE),
]:
):
result = await eval_fn(answer)
assert result == expected

Expand Down
14 changes: 7 additions & 7 deletions uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 56f50b5

Please sign in to comment.