From 3ccf592572efc441dc6a1cd849d0f1cd40452415 Mon Sep 17 00:00:00 2001 From: Evan Griffiths <56087052+evangriffiths@users.noreply.github.com> Date: Thu, 4 Apr 2024 09:20:27 +0100 Subject: [PATCH] Make KnownOutcomeAgent cheaper for unknown outcomes (#46) * Make KnownOutcomeAgent cheaper for unknown outcomes --------- Co-authored-by: Peter Jung --- .../agents/known_outcome_agent/benchmark.py | 13 ++++--- .../agents/known_outcome_agent/deploy.py | 26 +++++++------ .../known_outcome_agent.py | 38 +++++++++++++++---- 3 files changed, 53 insertions(+), 24 deletions(-) diff --git a/prediction_market_agent/agents/known_outcome_agent/benchmark.py b/prediction_market_agent/agents/known_outcome_agent/benchmark.py index cb122c4..5fe9008 100644 --- a/prediction_market_agent/agents/known_outcome_agent/benchmark.py +++ b/prediction_market_agent/agents/known_outcome_agent/benchmark.py @@ -32,7 +32,9 @@ def to_market(self) -> AgentMarket: id=self.question, question=self.question, p_yes=Probability( - self.result.to_p_yes() if self.result != Result.UNKNOWN else 0.5 + self.result.to_p_yes() + if self.result != Result.KNOWN_UNKNOWABLE + else 0.5 ), volume=None, created_time=None, @@ -60,7 +62,8 @@ def predict(self, market_question: str) -> Prediction: question=market_question, max_tries=self.max_tries, ) - if answer.result == Result.UNKNOWN: + print(f"Answered {market_question=} with {answer.result=}, {answer.reasoning=}") + if not answer.has_known_result(): return Prediction( is_predictable=False, outcome_prediction=None, @@ -127,17 +130,17 @@ def predict(self, market_question: str) -> Prediction: ), QuestionWithKnownOutcome( question="Will Lewis Hamilton win the 2024/2025 F1 drivers champtionship?", - result=Result.UNKNOWN, + result=Result.KNOWN_UNKNOWABLE, notes="Outcome is uncertain.", ), QuestionWithKnownOutcome( question="Will the cost of grain in the Spain increase by 20% by 19 July 2024?", - result=Result.UNKNOWN, + result=Result.KNOWN_UNKNOWABLE, notes="Outcome is uncertain.", ), QuestionWithKnownOutcome( question="Will over 360 pople have died while climbing Mount Everest by 1st Jan 2028?", - result=Result.UNKNOWN, + result=Result.KNOWN_UNKNOWABLE, notes="Outcome is uncertain.", ), ] diff --git a/prediction_market_agent/agents/known_outcome_agent/deploy.py b/prediction_market_agent/agents/known_outcome_agent/deploy.py index ced4f78..deb9629 100644 --- a/prediction_market_agent/agents/known_outcome_agent/deploy.py +++ b/prediction_market_agent/agents/known_outcome_agent/deploy.py @@ -1,4 +1,5 @@ import getpass +import random from decimal import Decimal from prediction_market_agent_tooling.config import APIKeys @@ -9,6 +10,7 @@ from prediction_market_agent_tooling.markets.data_models import BetAmount, Currency from prediction_market_agent_tooling.markets.markets import MarketType from prediction_market_agent_tooling.tools.utils import ( + check_not_none, get_current_git_commit_sha, get_current_git_url, ) @@ -16,7 +18,6 @@ from prediction_market_agent.agents.known_outcome_agent.known_outcome_agent import ( Result, get_known_outcome, - has_question_event_happened_in_the_past, ) @@ -42,17 +43,20 @@ def pick_markets(self, markets: list[AgentMarket]) -> list[AgentMarket]: print( f"Skipping market {market.id=} {market.question=}, because it is already saturated." ) - continue - - # TODO it is currently expensive and slow to run the full evaluation - # on all markets, so we only run it on markets that address events - # that have already happened in the past. - if has_question_event_happened_in_the_past( - model=self.model, question=market.question - ): - print(f"Picking market {market.id=} {market.question=}") + else: picked_markets.append(market) + # If all markets have a closing time set, pick the earliest closing. + # Otherwise pick randomly. + N_TO_PICK = 5 + if all(market.close_time for market in picked_markets): + picked_markets = sorted( + picked_markets, key=lambda market: check_not_none(market.close_time) + )[:N_TO_PICK] + else: + picked_markets = random.sample( + picked_markets, min(len(picked_markets), N_TO_PICK) + ) return picked_markets def answer_binary_market(self, market: AgentMarket) -> bool | None: @@ -67,7 +71,7 @@ def answer_binary_market(self, market: AgentMarket) -> bool | None: f"Error: Failed to predict market {market.id=} {market.question=}: {e}" ) answer = None - if answer and answer.has_known_outcome(): + if answer and answer.has_known_result(): print( f"Picking market {market.id=} {market.question=} with answer {answer.result=}" ) diff --git a/prediction_market_agent/agents/known_outcome_agent/known_outcome_agent.py b/prediction_market_agent/agents/known_outcome_agent/known_outcome_agent.py index 1c86f68..6833cbc 100644 --- a/prediction_market_agent/agents/known_outcome_agent/known_outcome_agent.py +++ b/prediction_market_agent/agents/known_outcome_agent/known_outcome_agent.py @@ -14,8 +14,21 @@ class Result(str, Enum): + """ + With perfect information, a binary question should have one of three answers: + YES, NO, or KNOWN_UNKNOWABLE: + + - Will the sun rise tomorrow? YES + - Will the Bradley Cooper win best actor at the 2024 oscars by 01/04/2024? NO (because the event has already happened, and Cillian Murphy won) + - Will over 360 pople have died while climbing Mount Everest by 1st Jan 2028? KNOWN_UNKNOWABLE (because the closing date has not happened yet, and we cannot predict the outcome with reasoanable certainty) + + but since the agent's information is based on web scraping, and is therefore + imperfect, we allow it to defer answering definitively via the UNKNOWN result. + """ + YES = "YES" NO = "NO" + KNOWN_UNKNOWABLE = "KNOWN_UNKNOWABLE" UNKNOWN = "UNKNOWN" def to_p_yes(self) -> float: @@ -34,13 +47,17 @@ def to_boolean(self) -> bool: else: raise ValueError("Unexpected result") + @property + def is_known(self) -> bool: + return self in [Result.YES, Result.NO] + class Answer(BaseModel): result: Result reasoning: str - def has_known_outcome(self) -> bool: - return self.result is not Result.UNKNOWN + def has_known_result(self) -> bool: + return self.result.is_known HAS_QUESTION_HAPPENED_IN_THE_PAST_PROMPT = """ @@ -72,7 +89,7 @@ def has_known_outcome(self) -> bool: You might generate the following search query: -"Champions League semi-finals draw 2024" +"Champions League semi-finals draw 2025" Answer with the single prompt only, and nothing else. """ @@ -100,20 +117,25 @@ def has_known_outcome(self) -> bool: where is a free text field containing your reasoning, and is a multiple-choice field containing only one of 'YES' (if the answer to the -question is yes), 'NO' (if the answer to the question is no), or 'UNKNOWN' if -you are unable to answer the question with a reasonable degree of certainty from -the web-scraped information. Your answer should only contain this json string, -and nothing else. +question is yes), 'NO' (if the answer to the question is no), 'KNOWN_UNKNOWABLE' +(if we can answer with a reasonable degree of certainty from the web-scraped +information that the question cannot be answered either way for the time being), +or 'UNKNOWN' if you are unable to give one of the above answers with a +reasonable degree of certainty from the web-scraped information. Your answer +should only contain this json string, and nothing else. If the question is of the format: "Will X happen by Y?" then the result should be as follows: - If X has already happened, the result is 'YES'. - If not-X has already happened, the result is 'NO'. - If X has been announced to happen after Y, result 'NO'. +- If you are confident that none of the above are the case, and the result will only be knowable in the future, or not at all, the result is 'KNOWN_UNKNOWABLE'. - Otherwise, the result is 'UNKNOWN'. -If the question is of the format: "Will X happen on Y?" +If the question is of the format: "Will X happen on Y?" then the result should +be as follows: - If something has happened that necessarily prevents X from happening on Y, the result is 'NO'. +- If you are confident that nothing has happened that necessarily prevents X from happening on Y, the result is 'KNOWN_UNKNOWABLE'. - Otherwise, the result is 'UNKNOWN'. """