examples/notebooks/retrieval_and_evaluation.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "OrfB2Z3LfT33"
      },
      "source": [
        "# Preparation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 765
        },
        "colab_type": "code",
        "id": "0m5M7WiZfNfk",
        "outputId": "7999e253-721f-4eaa-d1eb-cd9ca1ee8c82"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting python-terrier\n",
            "  Cloning https://github.com/terrier-org/pyterrier.git to /tmp/pip-install-_cprmknv/python-terrier\n",
            "  Running command git clone -q https://github.com/terrier-org/pyterrier.git /tmp/pip-install-_cprmknv/python-terrier\n",
            "Collecting pyjnius~=1.3.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d8/50/098cb5fb76fb7c7d99d403226a2a63dcbfb5c129b71b7d0f5200b05de1f0/pyjnius-1.3.0-cp36-cp36m-manylinux2010_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 2.8MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: numpy in /usr/local/lib/python3.6/dist-packages (from python-terrier) (1.18.4)\n",
            "Requirement already satisfied, skipping upgrade: pandas in /usr/local/lib/python3.6/dist-packages (from python-terrier) (1.0.4)\n",
            "Collecting wget\n",
            "  Downloading https://files.pythonhosted.org/packages/47/6a/62e288da7bcda82b935ff0c6cfe542970f04e29c756b0e147251b2fb251f/wget-3.2.zip\n",
            "Collecting pytrec_eval\n",
            "  Downloading https://files.pythonhosted.org/packages/36/0a/5809ba805e62c98f81e19d6007132712945c78e7612c11f61bac76a25ba3/pytrec_eval-0.4.tar.gz\n",
            "Requirement already satisfied, skipping upgrade: tqdm in /usr/local/lib/python3.6/dist-packages (from python-terrier) (4.41.1)\n",
            "Collecting matchpy\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/47/95/d265b944ce391bb2fa9982d7506bbb197bb55c5088ea74448a5ffcaeefab/matchpy-0.5.1-py3-none-any.whl (67kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 7.0MB/s \n",
            "\u001b[?25hCollecting deprecation\n",
            "  Downloading https://files.pythonhosted.org/packages/02/c3/253a89ee03fc9b9682f1541728eb66db7db22148cd94f89ab22528cd1e1b/deprecation-2.1.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied, skipping upgrade: six>=1.7.0 in /usr/local/lib/python3.6/dist-packages (from pyjnius~=1.3.0->python-terrier) (1.12.0)\n",
            "Requirement already satisfied, skipping upgrade: cython in /usr/local/lib/python3.6/dist-packages (from pyjnius~=1.3.0->python-terrier) (0.29.19)\n",
            "Requirement already satisfied, skipping upgrade: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas->python-terrier) (2018.9)\n",
            "Requirement already satisfied, skipping upgrade: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas->python-terrier) (2.8.1)\n",
            "Collecting hopcroftkarp<2.0,>=1.2\n",
            "  Downloading https://files.pythonhosted.org/packages/6b/56/7b03eba3c43008c490c9d52e69ea5334b65955f66836eb4f1962f3b0d421/hopcroftkarp-1.2.5.tar.gz\n",
            "Collecting multiset<3.0,>=2.0\n",
            "  Downloading https://files.pythonhosted.org/packages/a8/12/813a649f5bc9801865dc6cda95b8f169f784d996322db192907ebe399064/multiset-2.1.1-py2.py3-none-any.whl\n",
            "Requirement already satisfied, skipping upgrade: packaging in /usr/local/lib/python3.6/dist-packages (from deprecation->python-terrier) (20.4)\n",
            "Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->deprecation->python-terrier) (2.4.7)\n",
            "Building wheels for collected packages: python-terrier, wget, pytrec-eval, hopcroftkarp\n",
            "  Building wheel for python-terrier (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for python-terrier: filename=python_terrier-0.1.3-cp36-none-any.whl size=29595 sha256=3416e32a8cab385ebee1c0aed4b2ed808a23ffd6f5bbb7d9e4fbc3053eed7944\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-ntw1yv5l/wheels/cc/bb/69/836d846a92c787b35ca6478119c0033762ab2b95d866eeb288\n",
            "  Building wheel for wget (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for wget: filename=wget-3.2-cp36-none-any.whl size=9682 sha256=0dcf3863d2c1950d4aa7c8f78f25ed15c0538c5a6f4ccb3f848515d5b30d1103\n",
            "  Stored in directory: /root/.cache/pip/wheels/40/15/30/7d8f7cea2902b4db79e3fea550d7d7b85ecb27ef992b618f3f\n",
            "  Building wheel for pytrec-eval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pytrec-eval: filename=pytrec_eval-0.4-cp36-cp36m-linux_x86_64.whl size=273831 sha256=5de1014fbf13ce80459977600e5daa8bb3c8a9563ca7844652432aedecccdf12\n",
            "  Stored in directory: /root/.cache/pip/wheels/58/30/73/8858a1b6e5e2674e2ea85c9904949c06addcf6fd34d59b5ea6\n",
            "  Building wheel for hopcroftkarp (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for hopcroftkarp: filename=hopcroftkarp-1.2.5-py2.py3-none-any.whl size=18092 sha256=ad0ae9f34d942f8c786dea98d1983411bb25ec6a13113ecb85bdd94a1947553e\n",
            "  Stored in directory: /root/.cache/pip/wheels/2b/e1/c9/1993c7f7f114b7d3fb2d3e895e02157a7ebf554861e9e54e01\n",
            "Successfully built python-terrier wget pytrec-eval hopcroftkarp\n",
            "Installing collected packages: pyjnius, wget, pytrec-eval, hopcroftkarp, multiset, matchpy, deprecation, python-terrier\n",
            "Successfully installed deprecation-2.1.0 hopcroftkarp-1.2.5 matchpy-0.5.1 multiset-2.1.1 pyjnius-1.3.0 python-terrier-0.1.3 pytrec-eval-0.4 wget-3.2\n"
          ]
        }
      ],
      "source": [
        "%pip install -q python-terrier"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "colab_type": "code",
        "id": "pEJSTrbdfdK8",
        "outputId": "fd413a1a-09d2-4e5e-c5a3-c32a9821f14c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "terrier-assemblies 5.2  jar-with-dependencies not found, downloading to /root/.pyterrier...\n",
            "Done\n",
            "terrier-python-helper 0.0.2  jar not found, downloading to /root/.pyterrier...\n",
            "Done\n"
          ]
        }
      ],
      "source": [
        "import pyterrier as pt"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "eseqUHlf2V-2"
      },
      "source": [
        "# Vaswani_NPL\n",
        "\n",
        "We're going to use a very old IR test collection called [Vaswani_NPL](http://ir.dcs.gla.ac.uk/resources/test_collections/npl/). This is included with Terrier, but we provide access here to pre-made indices, along with the topics and qrels:\n",
        " "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "r6tA_Y2325Cw"
      },
      "outputs": [],
      "source": [
        "vaswani_dataset = pt.datasets.get_dataset(\"vaswani\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "IW4bzt40uVKR"
      },
      "source": [
        "# Load an existing index"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "colab_type": "code",
        "id": "SLGyh2OkuaXd",
        "outputId": "8662508d-251c-4e5e-b2f2-a71de781ee24"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Downloading vaswani index to /root/.pyterrier/corpora/vaswani/index\n",
            "Number of documents: 11429\n",
            "Number of terms: 7756\n",
            "Number of fields: 0\n",
            "Field names: []\n",
            "Number of tokens: 271581\n",
            "\n"
          ]
        }
      ],
      "source": [
        "indexref = vaswani_dataset.get_index()\n",
        "index = pt.IndexFactory.of(indexref)\n",
        "\n",
        "print(index.getCollectionStatistics().toString())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "rzW2v-qchgIF"
      },
      "source": [
        "# Retrieval"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "aHWBJ6NXlC1a"
      },
      "source": [
        "Normally, we would use pt.io.read_topics(topics_path) to parse a topics file. \n",
        "``` python\n",
        "topics_path = \"./query-text.trec\"\n",
        "topics = pt.io.read_topics(topics_path)\n",
        "```\n",
        "\n",
        "However, the pt.dataset gives the topics and qrels readily-parsed:\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        },
        "colab_type": "code",
        "id": "qR5fkwEglDIA",
        "outputId": "0a69c35a-3125-4e35-de45-a72c34ad49d4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Downloading vaswani topics to /root/.pyterrier/corpora/vaswani/query-text.trec\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>qid</th>\n",
              "      <th>query</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>measurement of dielectric constant of liquids ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>mathematical analysis and design details of wa...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>use of digital computers in the design of band...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>systems of data coding for information transfer</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>use of programs in engineering testing of comp...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  qid                                              query\n",
              "0   1  measurement of dielectric constant of liquids ...\n",
              "1   2  mathematical analysis and design details of wa...\n",
              "2   3  use of digital computers in the design of band...\n",
              "3   4    systems of data coding for information transfer\n",
              "4   5  use of programs in engineering testing of comp..."
            ]
          },
          "execution_count": 5,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topics = vaswani_dataset.get_topics()\n",
        "topics.head(5)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "4-fwnGg-lIzM"
      },
      "source": [
        "Create Retriever object\n",
        "\n",
        "You can optionally set the controls and the properties by passing a dictionary to the 'controls' and 'properties' arguments\n",
        "or by calling setControl or setControls methods on a created object, or use the default controls.\n",
        "\n",
        "Then call the transform method on the created object with the topics as an argument"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "wPG8PBwSlCuL"
      },
      "outputs": [],
      "source": [
        "retr = pt.terrier.Retriever(index, controls = {\"wmodel\": \"TF_IDF\"})\n",
        "\n",
        "retr.setControl(\"wmodel\", \"TF_IDF\")\n",
        "retr.setControls({\"wmodel\": \"TF_IDF\"})\n",
        "\n",
        "res=retr.transform(topics)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "colab_type": "code",
        "id": "uFGQXsJU5E9Q",
        "outputId": "ad84cb83-d249-4081-f12a-713b9bd90e1a"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>qid</th>\n",
              "      <th>docid</th>\n",
              "      <th>docno</th>\n",
              "      <th>rank</th>\n",
              "      <th>score</th>\n",
              "      <th>query</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>8171</td>\n",
              "      <td>8172</td>\n",
              "      <td>0</td>\n",
              "      <td>13.746087</td>\n",
              "      <td>measurement of dielectric constant of liquids ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>9880</td>\n",
              "      <td>9881</td>\n",
              "      <td>1</td>\n",
              "      <td>12.352666</td>\n",
              "      <td>measurement of dielectric constant of liquids ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>5501</td>\n",
              "      <td>5502</td>\n",
              "      <td>2</td>\n",
              "      <td>12.178153</td>\n",
              "      <td>measurement of dielectric constant of liquids ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>1501</td>\n",
              "      <td>1502</td>\n",
              "      <td>3</td>\n",
              "      <td>10.993585</td>\n",
              "      <td>measurement of dielectric constant of liquids ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>9858</td>\n",
              "      <td>9859</td>\n",
              "      <td>4</td>\n",
              "      <td>10.271452</td>\n",
              "      <td>measurement of dielectric constant of liquids ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91925</th>\n",
              "      <td>93</td>\n",
              "      <td>2226</td>\n",
              "      <td>2227</td>\n",
              "      <td>995</td>\n",
              "      <td>4.904950</td>\n",
              "      <td>high frequency oscillators using transistors t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91926</th>\n",
              "      <td>93</td>\n",
              "      <td>6898</td>\n",
              "      <td>6899</td>\n",
              "      <td>996</td>\n",
              "      <td>4.899385</td>\n",
              "      <td>high frequency oscillators using transistors t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91927</th>\n",
              "      <td>93</td>\n",
              "      <td>3473</td>\n",
              "      <td>3474</td>\n",
              "      <td>997</td>\n",
              "      <td>4.898796</td>\n",
              "      <td>high frequency oscillators using transistors t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91928</th>\n",
              "      <td>93</td>\n",
              "      <td>3187</td>\n",
              "      <td>3188</td>\n",
              "      <td>998</td>\n",
              "      <td>4.893073</td>\n",
              "      <td>high frequency oscillators using transistors t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91929</th>\n",
              "      <td>93</td>\n",
              "      <td>3158</td>\n",
              "      <td>3159</td>\n",
              "      <td>999</td>\n",
              "      <td>4.892029</td>\n",
              "      <td>high frequency oscillators using transistors t...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>91930 rows × 6 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "      qid  docid  ...      score                                              query\n",
              "0       1   8171  ...  13.746087  measurement of dielectric constant of liquids ...\n",
              "1       1   9880  ...  12.352666  measurement of dielectric constant of liquids ...\n",
              "2       1   5501  ...  12.178153  measurement of dielectric constant of liquids ...\n",
              "3       1   1501  ...  10.993585  measurement of dielectric constant of liquids ...\n",
              "4       1   9858  ...  10.271452  measurement of dielectric constant of liquids ...\n",
              "...    ..    ...  ...        ...                                                ...\n",
              "91925  93   2226  ...   4.904950  high frequency oscillators using transistors t...\n",
              "91926  93   6898  ...   4.899385  high frequency oscillators using transistors t...\n",
              "91927  93   3473  ...   4.898796  high frequency oscillators using transistors t...\n",
              "91928  93   3187  ...   4.893073  high frequency oscillators using transistors t...\n",
              "91929  93   3158  ...   4.892029  high frequency oscillators using transistors t...\n",
              "\n",
              "[91930 rows x 6 columns]"
            ]
          },
          "execution_count": 7,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "res"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "lAdFiXUJmDsc"
      },
      "source": [
        "You can also query simple string."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 493
        },
        "colab_type": "code",
        "id": "onRMf1dyWNlO",
        "outputId": "bad927e9-9bec-48a7-83ef-77f8d81b187a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    qid  docid  docno  rank     score  query\n",
            "0     1  10808  10809     0  5.537595  Light\n",
            "1     1  11231  11232     1  5.535640  Light\n",
            "2     1  11066  11067     2  5.497895  Light\n",
            "3     1   5995   5996     3  5.486707  Light\n",
            "4     1   4460   4461     4  5.464468  Light\n",
            "..   ..    ...    ...   ...       ...    ...\n",
            "120   1   4820   4821   120  1.964441  Light\n",
            "121   1   9836   9837   121  1.927833  Light\n",
            "122   1   7213   7214   122  1.910036  Light\n",
            "123   1   6177   6178   123  1.892565  Light\n",
            "124   1   7777   7778   124  1.251497  Light\n",
            "\n",
            "[125 rows x 6 columns]\n",
            "    qid  docid  docno  rank     score  query\n",
            "0     1  10808  10809     0  5.537595  Light\n",
            "1     1  11231  11232     1  5.535640  Light\n",
            "2     1  11066  11067     2  5.497895  Light\n",
            "3     1   5995   5996     3  5.486707  Light\n",
            "4     1   4460   4461     4  5.464468  Light\n",
            "..   ..    ...    ...   ...       ...    ...\n",
            "211   2   6374   6375    86  2.505309  Sound\n",
            "212   2   1695   1696    87  2.505309  Sound\n",
            "213   2   6546   6547    88  2.396764  Sound\n",
            "214   2   7859   7860    89  2.321334  Sound\n",
            "215   2   8492   8493    90  2.100934  Sound\n",
            "\n",
            "[216 rows x 6 columns]\n"
          ]
        }
      ],
      "source": [
        "retr.search(\"Light\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "1k5DvZHQjEhU"
      },
      "source": [
        "You can save the result to a file by using `pt.io.write_results(result, path)`"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "g9qGpGSxjOHi"
      },
      "outputs": [],
      "source": [
        "pt.io.write_results(res,\"result1.res\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "JeNYoPbrj7FN"
      },
      "source": [
        "# Evaluation"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "qwYjiURXkXJp"
      },
      "source": [
        "Similarly, if working with a local test collection, we can use pt.io.read_qrels(qrels_path) to parse a qrels file:\n",
        "```python\n",
        "qrels_path=(\"./qrels\")\n",
        "qrels = pt.io.read_qrels(qrels_path)\n",
        "```\n",
        "\n",
        "However, for the Vaswani dataset, the qrels are provided ready-to-do:\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "colab_type": "code",
        "id": "uvw_CXrak5MQ",
        "outputId": "5d138c87-20cb-4f94-dac9-92c3eb0b26da"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Downloading vaswani qrels to /root/.pyterrier/corpora/vaswani/qrels\n"
          ]
        }
      ],
      "source": [
        "qrels = vaswani_dataset.get_qrels()"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "ON8yEcWukIIA"
      },
      "source": [
        "Use `pt.Evaluate(results, qrels)` to evaluate the results    \n",
        "Args:    \n",
        "metrics, `default = [\"map\", ndcg\"]`, select the evaluation metrics    \n",
        "perquery, `default = False`, select whether to show the mean of the metrics or the metrics for each query"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "colab_type": "code",
        "id": "LXfETppwj55p",
        "outputId": "909797b7-791b-46f4-8559-0b286e5bcacb"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'map': 0.29090543005529873, 'ndcg': 0.6153667539666847}"
            ]
          },
          "execution_count": 11,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "eval = pt.Evaluate(res,qrels)\n",
        "eval"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "TiDVb4iG4EyH"
      },
      "source": [
        "We can also ask for per-query results."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "colab_type": "code",
        "id": "IhMR2YurnS1Y",
        "outputId": "560159dd-10b8-44ba-a4ae-389faab05ef9"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'1': {'map': 0.2688603632606692},\n",
              " '10': {'map': 0.1214856066519094},\n",
              " '11': {'map': 0.06799761023743447},\n",
              " '12': {'map': 0.2093716360982601},\n",
              " '13': {'map': 0.26945162856284827},\n",
              " '14': {'map': 0.3164929260069987},\n",
              " '15': {'map': 0.17479160483981196},\n",
              " '16': {'map': 0.07376769675516924},\n",
              " '17': {'map': 0.3965636483508813},\n",
              " '18': {'map': 0.16354405989238738},\n",
              " '19': {'map': 0.44669647488527836},\n",
              " '2': {'map': 0.056448212440045914},\n",
              " '20': {'map': 0.22061080821325293},\n",
              " '21': {'map': 0.5395186359625185},\n",
              " '22': {'map': 0.3874015813665481},\n",
              " '23': {'map': 0.34623362970302457},\n",
              " '24': {'map': 0.19184305434732396},\n",
              " '25': {'map': 0.17181819840273246},\n",
              " '26': {'map': 0.46224321892311115},\n",
              " '27': {'map': 0.3332977158611145},\n",
              " '28': {'map': 0.3248793014207182},\n",
              " '29': {'map': 0.3678434174356832},\n",
              " '3': {'map': 0.23945401361406524},\n",
              " '30': {'map': 0.3740405619725896},\n",
              " '31': {'map': 0.3659688796052433},\n",
              " '32': {'map': 0.5449193708233969},\n",
              " '33': {'map': 0.16758096895311753},\n",
              " '34': {'map': 0.09475121050961019},\n",
              " '35': {'map': 0.3161795835811207},\n",
              " '36': {'map': 0.06240987386368677},\n",
              " '37': {'map': 0.5629044478591622},\n",
              " '38': {'map': 0.4638100016959579},\n",
              " '39': {'map': 0.413195706028076},\n",
              " '4': {'map': 0.4939494140851607},\n",
              " '40': {'map': 0.5156482922211466},\n",
              " '41': {'map': 0.13989156957293147},\n",
              " '42': {'map': 0.5901172920429301},\n",
              " '43': {'map': 0.3976663487840697},\n",
              " '44': {'map': 0.5942519338041726},\n",
              " '45': {'map': 0.5193686944361203},\n",
              " '46': {'map': 0.5428678808879653},\n",
              " '47': {'map': 0.47518054950991434},\n",
              " '48': {'map': 0.6269841269841269},\n",
              " '49': {'map': 0.5117016956036146},\n",
              " '5': {'map': 0.0},\n",
              " '50': {'map': 0.02702702702702703},\n",
              " '51': {'map': 0.4066181375307124},\n",
              " '52': {'map': 0.38004411122245385},\n",
              " '53': {'map': 0.09166387119377388},\n",
              " '54': {'map': 0.4328492784371206},\n",
              " '55': {'map': 0.16575528585039312},\n",
              " '56': {'map': 0.33103980418112827},\n",
              " '57': {'map': 0.10598901358272976},\n",
              " '58': {'map': 0.041942795637681884},\n",
              " '59': {'map': 0.00684931506849315},\n",
              " '6': {'map': 0.2421600270476016},\n",
              " '60': {'map': 0.2661290322580645},\n",
              " '61': {'map': 0.2325178435051701},\n",
              " '62': {'map': 0.45097069189164174},\n",
              " '63': {'map': 0.4636814137458281},\n",
              " '64': {'map': 0.06882148846081122},\n",
              " '65': {'map': 0.4332906530089628},\n",
              " '66': {'map': 0.02541169246210568},\n",
              " '67': {'map': 0.0906611872453528},\n",
              " '68': {'map': 0.1407544607599312},\n",
              " '69': {'map': 0.43466511590479573},\n",
              " '7': {'map': 0.5674516736006812},\n",
              " '70': {'map': 0.11692282520484829},\n",
              " '71': {'map': 0.2986642285293222},\n",
              " '72': {'map': 0.4846427166744158},\n",
              " '73': {'map': 0.4518459686644996},\n",
              " '74': {'map': 0.2160902685061814},\n",
              " '75': {'map': 0.7575150494802282},\n",
              " '76': {'map': 0.3368632248482061},\n",
              " '77': {'map': 0.4056841177263038},\n",
              " '78': {'map': 0.1207974393458629},\n",
              " '79': {'map': 0.09493367806275874},\n",
              " '8': {'map': 0.5},\n",
              " '80': {'map': 0.007144142221420812},\n",
              " '81': {'map': 0.30401935021500237},\n",
              " '82': {'map': 0.2396311160423612},\n",
              " '83': {'map': 0.23493235154351588},\n",
              " '84': {'map': 0.32846055524463724},\n",
              " '85': {'map': 0.0059122070286004354},\n",
              " '86': {'map': 0.08178341113533369},\n",
              " '87': {'map': 0.22253254033872852},\n",
              " '88': {'map': 0.060998128884790115},\n",
              " '89': {'map': 0.08737273117424477},\n",
              " '9': {'map': 0.5222222222222223},\n",
              " '90': {'map': 0.16870990966976288},\n",
              " '91': {'map': 0.3564038634257101},\n",
              " '92': {'map': 0.08935884162889926},\n",
              " '93': {'map': 0.23046674164654785}}"
            ]
          },
          "execution_count": 12,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "eval = pt.Evaluate(res,qrels,metrics=[\"map\"], perquery=True)\n",
        "eval"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "N9ZSXTjS3_Z9"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "Retrieval and Evaluation.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.9.6"
    },
    "vscode": {
      "interpreter": {
        "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}