Make KnownOutcomeAgent cheaper for unknown outcomes (#46)

* Make KnownOutcomeAgent cheaper for unknown outcomes
---------

Co-authored-by: Peter Jung <peter@jung.ninja>

prediction_market_agent/agents/known_outcome_agent/benchmark.py

@@ -32,7 +32,9 @@ def to_market(self) -> AgentMarket:
             id=self.question,
             question=self.question,
             p_yes=Probability(
-                self.result.to_p_yes() if self.result != Result.UNKNOWN else 0.5
+                self.result.to_p_yes()
+                if self.result != Result.KNOWN_UNKNOWABLE
+                else 0.5
             ),
             volume=None,
             created_time=None,
@@ -60,7 +62,8 @@ def predict(self, market_question: str) -> Prediction:
             question=market_question,
             max_tries=self.max_tries,
         )
-        if answer.result == Result.UNKNOWN:
+        print(f"Answered {market_question=} with {answer.result=}, {answer.reasoning=}")
+        if not answer.has_known_result():
             return Prediction(
                 is_predictable=False,
                 outcome_prediction=None,
@@ -127,17 +130,17 @@ def predict(self, market_question: str) -> Prediction:
         ),
         QuestionWithKnownOutcome(
             question="Will Lewis Hamilton win the 2024/2025 F1 drivers champtionship?",
-            result=Result.UNKNOWN,
+            result=Result.KNOWN_UNKNOWABLE,
             notes="Outcome is uncertain.",
         ),
         QuestionWithKnownOutcome(
             question="Will the cost of grain in the Spain increase by 20% by 19 July 2024?",
-            result=Result.UNKNOWN,
+            result=Result.KNOWN_UNKNOWABLE,
             notes="Outcome is uncertain.",
         ),
         QuestionWithKnownOutcome(
             question="Will over 360 pople have died while climbing Mount Everest by 1st Jan 2028?",
-            result=Result.UNKNOWN,
+            result=Result.KNOWN_UNKNOWABLE,
             notes="Outcome is uncertain.",
         ),
     ]

prediction_market_agent/agents/known_outcome_agent/deploy.py

@@ -1,4 +1,5 @@
 import getpass
+import random
 from decimal import Decimal
 
 from prediction_market_agent_tooling.config import APIKeys
@@ -9,14 +10,14 @@
 from prediction_market_agent_tooling.markets.data_models import BetAmount, Currency
 from prediction_market_agent_tooling.markets.markets import MarketType
 from prediction_market_agent_tooling.tools.utils import (
+    check_not_none,
     get_current_git_commit_sha,
     get_current_git_url,
 )
 
 from prediction_market_agent.agents.known_outcome_agent.known_outcome_agent import (
     Result,
     get_known_outcome,
-    has_question_event_happened_in_the_past,
 )
 
 
@@ -42,17 +43,20 @@ def pick_markets(self, markets: list[AgentMarket]) -> list[AgentMarket]:
                 print(
                     f"Skipping market {market.id=} {market.question=}, because it is already saturated."
                 )
-                continue
-
-            # TODO it is currently expensive and slow to run the full evaluation
-            # on all markets, so we only run it on markets that address events
-            # that have already happened in the past.
-            if has_question_event_happened_in_the_past(
-                model=self.model, question=market.question
-            ):
-                print(f"Picking market {market.id=} {market.question=}")
+            else:
                 picked_markets.append(market)
 
+        # If all markets have a closing time set, pick the earliest closing.
+        # Otherwise pick randomly.
+        N_TO_PICK = 5
+        if all(market.close_time for market in picked_markets):
+            picked_markets = sorted(
+                picked_markets, key=lambda market: check_not_none(market.close_time)
+            )[:N_TO_PICK]
+        else:
+            picked_markets = random.sample(
+                picked_markets, min(len(picked_markets), N_TO_PICK)
+            )
         return picked_markets
 
     def answer_binary_market(self, market: AgentMarket) -> bool | None:
@@ -67,7 +71,7 @@ def answer_binary_market(self, market: AgentMarket) -> bool | None:
                 f"Error: Failed to predict market {market.id=} {market.question=}: {e}"
             )
             answer = None
-        if answer and answer.has_known_outcome():
+        if answer and answer.has_known_result():
             print(
                 f"Picking market {market.id=} {market.question=} with answer {answer.result=}"
             )

prediction_market_agent/agents/known_outcome_agent/known_outcome_agent.py

@@ -14,8 +14,21 @@
 
 
 class Result(str, Enum):
+    """
+    With perfect information, a binary question should have one of three answers:
+    YES, NO, or KNOWN_UNKNOWABLE:
+
+    - Will the sun rise tomorrow? YES
+    - Will the Bradley Cooper win best actor at the 2024 oscars by 01/04/2024? NO (because the event has already happened, and Cillian Murphy won)
+    - Will over 360 pople have died while climbing Mount Everest by 1st Jan 2028? KNOWN_UNKNOWABLE (because the closing date has not happened yet, and we cannot predict the outcome with reasoanable certainty)
+
+    but since the agent's information is based on web scraping, and is therefore
+    imperfect, we allow it to defer answering definitively via the UNKNOWN result.
+    """
+
     YES = "YES"
     NO = "NO"
+    KNOWN_UNKNOWABLE = "KNOWN_UNKNOWABLE"
     UNKNOWN = "UNKNOWN"
 
     def to_p_yes(self) -> float:
@@ -34,13 +47,17 @@ def to_boolean(self) -> bool:
         else:
             raise ValueError("Unexpected result")
 
+    @property
+    def is_known(self) -> bool:
+        return self in [Result.YES, Result.NO]
+
 
 class Answer(BaseModel):
     result: Result
     reasoning: str
 
-    def has_known_outcome(self) -> bool:
-        return self.result is not Result.UNKNOWN
+    def has_known_result(self) -> bool:
+        return self.result.is_known
 
 
 HAS_QUESTION_HAPPENED_IN_THE_PAST_PROMPT = """
@@ -72,7 +89,7 @@ def has_known_outcome(self) -> bool:
 
 You might generate the following search query:
 
-"Champions League semi-finals draw 2024"
+"Champions League semi-finals draw 2025"
 
 Answer with the single prompt only, and nothing else.
 """
@@ -100,20 +117,25 @@ def has_known_outcome(self) -> bool:
 
 where <REASONING> is a free text field containing your reasoning, and <RESULT>
 is a multiple-choice field containing only one of 'YES' (if the answer to the
-question is yes), 'NO' (if the answer to the question is no), or 'UNKNOWN' if
-you are unable to answer the question with a reasonable degree of certainty from
-the web-scraped information. Your answer should only contain this json string,
-and nothing else.
+question is yes), 'NO' (if the answer to the question is no), 'KNOWN_UNKNOWABLE'
+(if we can answer with a reasonable degree of certainty from the web-scraped
+information that the question cannot be answered either way for the time being),
+or 'UNKNOWN' if you are unable to give one of the above answers with a
+reasonable degree of certainty from the web-scraped information. Your answer
+should only contain this json string, and nothing else.
 
 If the question is of the format: "Will X happen by Y?" then the result should
 be as follows:
 - If X has already happened, the result is 'YES'.
 - If not-X has already happened, the result is 'NO'.
 - If X has been announced to happen after Y, result 'NO'.
+- If you are confident that none of the above are the case, and the result will only be knowable in the future, or not at all, the result is 'KNOWN_UNKNOWABLE'.
 - Otherwise, the result is 'UNKNOWN'.
 
-If the question is of the format: "Will X happen on Y?"
+If the question is of the format: "Will X happen on Y?" then the result should
+be as follows:
 - If something has happened that necessarily prevents X from happening on Y, the result is 'NO'.
+- If you are confident that nothing has happened that necessarily prevents X from happening on Y, the result is 'KNOWN_UNKNOWABLE'.
 - Otherwise, the result is 'UNKNOWN'.
 """