zero_shot_hypothesis_based_approach.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "zero_shot_hypothesis_based_approach.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "OKjVBISRc4kD"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "do-TXGBemGgH"
      },
      "source": [
        "# Setup "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5WsITUAnzvFl"
      },
      "source": [
        "Download the Task data and evaluation scripts"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qq3qhQdpl-1-",
        "outputId": "5af7bd5c-3b8e-4955-d1e7-06939fea8140"
      },
      "source": [
        "!git clone https://github.com/H-TayyarMadabushi/SemEval_2022_Task2-idiomaticity.git"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'SemEval_2022_Task2-idiomaticity'...\n",
            "remote: Enumerating objects: 92, done.\u001b[K\n",
            "remote: Counting objects: 100% (92/92), done.\u001b[K\n",
            "remote: Compressing objects: 100% (75/75), done.\u001b[K\n",
            "remote: Total 92 (delta 31), reused 67 (delta 15), pack-reused 0\u001b[K\n",
            "Unpacking objects: 100% (92/92), done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u-0POB9tzfNx"
      },
      "source": [
        "Download the “AStitchInLanguageModels” code which we make use of. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "affNQCRktdx4",
        "outputId": "0d84b845-570a-4b8c-bdaa-62c7dccaf7ae"
      },
      "source": [
        "!git clone https://github.com/H-TayyarMadabushi/AStitchInLanguageModels.git"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'AStitchInLanguageModels'...\n",
            "remote: Enumerating objects: 1030, done.\u001b[K\n",
            "remote: Counting objects: 100% (1030/1030), done.\u001b[K\n",
            "remote: Compressing objects: 100% (772/772), done.\u001b[K\n",
            "remote: Total 1030 (delta 382), reused 803 (delta 202), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (1030/1030), 79.86 MiB | 13.34 MiB/s, done.\n",
            "Resolving deltas: 100% (382/382), done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "60w-An2vzikk"
      },
      "source": [
        "Download and install an editable version of huggingfaces transformers. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m8BhcLYcmVvd",
        "outputId": "7df86f89-3b09-43aa-8ac4-67bc228c7420"
      },
      "source": [
        "!git clone https://github.com/huggingface/transformers.git\n",
        "%cd transformers/\n",
        "!pip install --editable .\n",
        "%cd /content/ "
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'transformers'...\n",
            "remote: Enumerating objects: 91488, done.\u001b[K\n",
            "remote: Counting objects: 100% (5/5), done.\u001b[K\n",
            "remote: Compressing objects: 100% (5/5), done.\u001b[K\n",
            "remote: Total 91488 (delta 0), reused 1 (delta 0), pack-reused 91483\u001b[K\n",
            "Receiving objects: 100% (91488/91488), 75.67 MiB | 14.39 MiB/s, done.\n",
            "Resolving deltas: 100% (66031/66031), done.\n",
            "/content/transformers\n",
            "Obtaining file:///content/transformers\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.14.0.dev0) (2019.12.20)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers==4.14.0.dev0) (4.8.2)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers==4.14.0.dev0) (4.62.3)\n",
            "Collecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.2.1-py3-none-any.whl (61 kB)\n",
            "\u001b[K     |████████████████████████████████| 61 kB 501 kB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.14.0.dev0) (1.19.5)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 8.5 MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "  Downloading sacremoses-0.0.46-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 48.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers==4.14.0.dev0) (2.23.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers==4.14.0.dev0) (3.4.0)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "  Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3 MB 26.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers==4.14.0.dev0) (21.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers==4.14.0.dev0) (3.10.0.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers==4.14.0.dev0) (3.0.6)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers==4.14.0.dev0) (3.6.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.14.0.dev0) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.14.0.dev0) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.14.0.dev0) (2021.10.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.14.0.dev0) (3.0.4)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.14.0.dev0) (7.1.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.14.0.dev0) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.14.0.dev0) (1.1.0)\n",
            "Installing collected packages: pyyaml, tokenizers, sacremoses, huggingface-hub, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "  Running setup.py develop for transformers\n",
            "Successfully installed huggingface-hub-0.2.1 pyyaml-6.0 sacremoses-0.0.46 tokenizers-0.10.3 transformers-4.14.0.dev0\n",
            "/content\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "huVMnwTSzmjJ"
      },
      "source": [
        "Required for run_glue ... "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2tsWits5tw1t",
        "outputId": "1d77cf46-454f-489d-a655-168a2ab06f12"
      },
      "source": [
        "## run_glue needs this. \n",
        "!pip install datasets"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting datasets\n",
            "  Downloading datasets-1.16.1-py3-none-any.whl (298 kB)\n",
            "\u001b[K     |████████████████████████████████| 298 kB 4.0 MB/s \n",
            "\u001b[?25hCollecting fsspec[http]>=2021.05.0\n",
            "  Downloading fsspec-2021.11.1-py3-none-any.whl (132 kB)\n",
            "\u001b[K     |████████████████████████████████| 132 kB 68.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets) (1.19.5)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.4)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl (243 kB)\n",
            "\u001b[K     |████████████████████████████████| 243 kB 53.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: huggingface-hub<1.0.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.2.1)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.12.2)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from datasets) (21.3)\n",
            "Requirement already satisfied: pyarrow!=4.0.0,>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (3.0.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from datasets) (4.8.2)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2.23.0)\n",
            "Collecting aiohttp\n",
            "  Downloading aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 56.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.1.5)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (4.62.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.10.0.2)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (6.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.4.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->datasets) (3.0.6)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2021.10.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Collecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.7.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (271 kB)\n",
            "\u001b[K     |████████████████████████████████| 271 kB 67.3 MB/s \n",
            "\u001b[?25hCollecting async-timeout<5.0,>=4.0.0a3\n",
            "  Downloading async_timeout-4.0.1-py3-none-any.whl (5.7 kB)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (21.2.0)\n",
            "Collecting multidict<7.0,>=4.5\n",
            "  Downloading multidict-5.2.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (160 kB)\n",
            "\u001b[K     |████████████████████████████████| 160 kB 70.6 MB/s \n",
            "\u001b[?25hCollecting asynctest==0.13.0\n",
            "  Downloading asynctest-0.13.0-py3-none-any.whl (26 kB)\n",
            "Collecting frozenlist>=1.1.1\n",
            "  Downloading frozenlist-1.2.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (192 kB)\n",
            "\u001b[K     |████████████████████████████████| 192 kB 53.5 MB/s \n",
            "\u001b[?25hCollecting aiosignal>=1.1.2\n",
            "  Downloading aiosignal-1.2.0-py3-none-any.whl (8.2 kB)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (2.0.8)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->datasets) (3.6.0)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2018.9)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
            "Installing collected packages: multidict, frozenlist, yarl, asynctest, async-timeout, aiosignal, fsspec, aiohttp, xxhash, datasets\n",
            "Successfully installed aiohttp-3.8.1 aiosignal-1.2.0 async-timeout-4.0.1 asynctest-0.13.0 datasets-1.16.1 frozenlist-1.2.0 fsspec-2021.11.1 multidict-5.2.0 xxhash-2.0.2 yarl-1.7.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-igYdTTgzp9e"
      },
      "source": [
        "Editable install requires runtime restart unless we do this. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uOuKplBmmbeB"
      },
      "source": [
        "import site\n",
        "site.main()\n"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "n3x5hKOBdBtd"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TvC8kAGNnKk_"
      },
      "source": [
        "# Imports and Helper functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aOw3MaG7nN77"
      },
      "source": [
        "import os\n",
        "import csv\n",
        "\n",
        "from pathlib import Path"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MzDtW9eXnOhG"
      },
      "source": [
        "def load_csv( path, delimiter=',' ) : \n",
        "  header = None\n",
        "  data   = list()\n",
        "  with open( path, encoding='utf-8') as csvfile:\n",
        "    reader = csv.reader( csvfile, delimiter=delimiter ) \n",
        "    for row in reader : \n",
        "      if header is None : \n",
        "        header = row\n",
        "        continue\n",
        "      data.append( row ) \n",
        "  return header, data\n"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WwtDsdtAnSZu"
      },
      "source": [
        "def write_csv( data, location ) : \n",
        "  with open( location, 'w', encoding='utf-8') as csvfile:\n",
        "    writer = csv.writer( csvfile ) \n",
        "    writer.writerows( data ) \n",
        "  print( \"Wrote {}\".format( location ) ) \n",
        "  return\n"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p9Io3D3_z4wt"
      },
      "source": [
        "The following function creates a submission file from the predictions output by run_glue (the text classification script from huggingface transformers - see below). \n",
        "\n",
        "Note that we set it up so we can load up results for only one setting. \n",
        "\n",
        "It requires as input the submission format file, which is available with the data. You can call this after completing each setting to load up results for both settings (see below).\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Re31vnLoQWww"
      },
      "source": [
        "def insert_to_submission_file( submission_format_file, input_file, prediction_format_file, setting ) :\n",
        "    submission_header, submission_content = load_csv( submission_format_file )\n",
        "    input_header     , input_data         = load_csv( input_file             )\n",
        "    prediction_header, prediction_data    = load_csv( prediction_format_file, '\\t' )\n",
        "\n",
        "    assert len( input_data ) == len( prediction_data )\n",
        "\n",
        "    ## submission_header ['ID', 'Language', 'Setting', 'Label']\n",
        "    ## input_header      ['label', 'sentence1' ]\n",
        "    ## prediction_header ['index', 'prediction']\n",
        "\n",
        "    prediction_data = list( reversed( prediction_data ) )\n",
        "\n",
        "    started_insert  = False\n",
        "    for elem in submission_content : \n",
        "        if elem[ submission_header.index( 'Setting' ) ] != setting :\n",
        "            if started_insert :\n",
        "                if len( prediction_data ) == 0 :\n",
        "                    break\n",
        "                else : \n",
        "                    raise Exception( \"Update should to contiguous ... something wrong.\" ) \n",
        "            continue\n",
        "        started_insert = True\n",
        "        elem[ submission_header.index( 'Label' ) ] = prediction_data.pop()[ prediction_header.index( 'prediction' ) ]\n",
        "\n",
        "    return [ submission_header ] + submission_content"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def _get_train_data( data_location, file_name, include_context, include_idiom ) :\n",
        "    \n",
        "    file_name = os.path.join( data_location, file_name ) \n",
        "\n",
        "    header, data = load_csv( file_name )\n",
        "\n",
        "    out_header = [ 'label', 'sentence1' ]\n",
        "    if include_idiom :\n",
        "        out_header = [ 'label', 'sentence1', 'sentence2' ]\n",
        "        \n",
        "    # ['DataID', 'Language', 'MWE', 'Setting', 'Previous', 'Target', 'Next', 'Label']\n",
        "    out_data = list()\n",
        "    for elem in data :\n",
        "        label     = elem[ header.index( 'Label'  ) ]\n",
        "        sentence1 = elem[ header.index( 'Target' ) ]\n",
        "        if include_context :\n",
        "            sentence1 = ' '.join( [ elem[ header.index( 'Previous' ) ], elem[ header.index( 'Target' ) ], elem[ header.index( 'Next' ) ] ] )\n",
        "        this_row = None\n",
        "        if not include_idiom :\n",
        "            this_row = [ label, sentence1 ] \n",
        "        else :\n",
        "            sentence2 = elem[ header.index( 'MWE' ) ]\n",
        "            this_row = [ label, sentence1, sentence2 ]\n",
        "        out_data.append( this_row )\n",
        "        assert len( out_header ) == len( this_row )\n",
        "    return [ out_header ] + out_data"
      ],
      "metadata": {
        "id": "K83KNRAKeSUz"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def _get_dev_eval_data( data_location, input_file_name, gold_file_name, include_context, include_idiom ) :\n",
        "\n",
        "    input_headers, input_data = load_csv( os.path.join( data_location, input_file_name ) )\n",
        "    gold_header  = gold_data = None\n",
        "    if not gold_file_name is None : \n",
        "        gold_header  , gold_data  = load_csv( os.path.join( data_location, gold_file_name  ) )\n",
        "        assert len( input_data ) == len( gold_data )\n",
        "\n",
        "    # ['ID', 'Language', 'MWE', 'Previous', 'Target', 'Next']\n",
        "    # ['ID', 'DataID', 'Language', 'Label']\n",
        "    \n",
        "    out_header = [ 'label', 'sentence1' ]\n",
        "    if include_idiom :\n",
        "        out_header = [ 'label', 'sentence1', 'sentence2' ]\n",
        "\n",
        "    out_data = list()\n",
        "    for index in range( len( input_data ) ) :\n",
        "        label = 1\n",
        "        if not gold_file_name is None : \n",
        "            this_input_id = input_data[ index ][ input_headers.index( 'ID' ) ]\n",
        "            this_gold_id  = gold_data [ index ][ gold_header  .index( 'ID' ) ]\n",
        "            assert this_input_id == this_gold_id\n",
        "            \n",
        "            label     = gold_data[ index ][ gold_header.index( 'Label'  ) ]\n",
        "            \n",
        "        elem      = input_data[ index ]\n",
        "        sentence1 = elem[ input_headers.index( 'Target' ) ]\n",
        "        if include_context :\n",
        "            sentence1 = ' '.join( [ elem[ input_headers.index( 'Previous' ) ], elem[ input_headers.index( 'Target' ) ], elem[ input_headers.index( 'Next' ) ] ] )\n",
        "        this_row = None\n",
        "        if not include_idiom :\n",
        "            this_row = [ label, sentence1 ] \n",
        "        else :\n",
        "            sentence2 = elem[ input_headers.index( 'MWE' ) ]\n",
        "            this_row = [ label, sentence1, sentence2 ]\n",
        "        assert len( out_header ) == len( this_row ) \n",
        "        out_data.append( this_row )\n",
        "        \n",
        "\n",
        "    return [ out_header ] + out_data\n"
      ],
      "metadata": {
        "id": "qax6SB9feUHM"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\"\"\"\n",
        "Based on the results presented in `AStitchInLanguageModels' we work with not including the idiom for the zero shot setting and including it in the one shot setting.\n",
        "\"\"\"\n",
        "def create_data( input_location, output_location ) :\n",
        "\n",
        "    \n",
        "    ## Zero shot data\n",
        "    train_data = _get_train_data(\n",
        "        data_location   = input_location,\n",
        "        file_name       = 'train_zero_shot.csv',\n",
        "        include_context = True,\n",
        "        include_idiom   = False\n",
        "    )\n",
        "    write_csv( train_data, os.path.join( output_location, 'ZeroShot', 'train.csv' ) )\n",
        "    \n",
        "    dev_data = _get_dev_eval_data(\n",
        "        data_location    = input_location,\n",
        "        input_file_name  = 'dev.csv',\n",
        "        gold_file_name   = 'dev_gold.csv', \n",
        "        include_context  = True,\n",
        "        include_idiom    = False\n",
        "    )        \n",
        "    write_csv( dev_data, os.path.join( output_location, 'ZeroShot', 'dev.csv' ) )\n",
        "    \n",
        "    eval_data = _get_dev_eval_data(\n",
        "        data_location    = input_location,\n",
        "        input_file_name  = 'eval.csv',\n",
        "        gold_file_name   = None , ## Don't have gold evaluation file -- submit to CodaLab\n",
        "        include_context  = True,\n",
        "        include_idiom    = False\n",
        "    )\n",
        "    write_csv( eval_data, os.path.join( output_location, 'ZeroShot', 'eval.csv' ) )\n",
        "\n",
        "\n",
        "    ## OneShot Data (combine both for training)\n",
        "    train_zero_data = _get_train_data(\n",
        "        data_location   = input_location,\n",
        "        file_name       = 'train_zero_shot.csv',\n",
        "        include_context = False,\n",
        "        include_idiom   = True\n",
        "    )\n",
        "    train_one_data = _get_train_data(\n",
        "        data_location   = input_location,\n",
        "        file_name       = 'train_one_shot.csv',\n",
        "        include_context = False,\n",
        "        include_idiom   = True\n",
        "    )\n",
        "\n",
        "    assert train_zero_data[0] == train_one_data[0] ## Headers\n",
        "    train_data = train_one_data + train_zero_data[1:]\n",
        "    write_csv( train_data, os.path.join( output_location, 'OneShot', 'train.csv' ) )\n",
        "    \n",
        "    dev_data = _get_dev_eval_data(\n",
        "        data_location    = input_location,\n",
        "        input_file_name  = 'dev.csv',\n",
        "        gold_file_name   = 'dev_gold.csv', \n",
        "        include_context  = False,\n",
        "        include_idiom    = True\n",
        "    )        \n",
        "    write_csv( dev_data, os.path.join( output_location, 'OneShot', 'dev.csv' ) )\n",
        "    \n",
        "    eval_data = _get_dev_eval_data(\n",
        "        data_location    = input_location,\n",
        "        input_file_name  = 'eval.csv',\n",
        "        gold_file_name   = None,\n",
        "        include_context  = False,\n",
        "        include_idiom    = True\n",
        "    )\n",
        "    write_csv( eval_data, os.path.join( output_location, 'OneShot', 'eval.csv' ) )\n",
        "\n",
        "    return"
      ],
      "metadata": {
        "id": "RibNDsYzeXFe"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "df = pd.read_csv(\"/content/SemEval_2022_Task2-idiomaticity/SubTaskA/Data/train_zero_shot.csv\")\n",
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 597
        },
        "id": "gXsVMI-beZDA",
        "outputId": "8d00dc16-8222-4842-cbb5-3c8491cf39c6"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>DataID</th>\n",
              "      <th>Language</th>\n",
              "      <th>MWE</th>\n",
              "      <th>Setting</th>\n",
              "      <th>Previous</th>\n",
              "      <th>Target</th>\n",
              "      <th>Next</th>\n",
              "      <th>Label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>train_zero_shot.EN.168.1</td>\n",
              "      <td>EN</td>\n",
              "      <td>double dutch</td>\n",
              "      <td>zero_shot</td>\n",
              "      <td>This inspired others to jump ropes as a leisur...</td>\n",
              "      <td>There are several theories behind the origin o...</td>\n",
              "      <td>The most popular theory states that “Double Du...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>train_zero_shot.EN.168.2</td>\n",
              "      <td>EN</td>\n",
              "      <td>double dutch</td>\n",
              "      <td>zero_shot</td>\n",
              "      <td>In the age of chivalry a man paid for the woma...</td>\n",
              "      <td>Double Dutch also derives from the same era, D...</td>\n",
              "      <td>There are many phrases that include the word: ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>train_zero_shot.EN.168.3</td>\n",
              "      <td>EN</td>\n",
              "      <td>double dutch</td>\n",
              "      <td>zero_shot</td>\n",
              "      <td>To her eternal credit, she kept both India and...</td>\n",
              "      <td>Since 1977 we have had a plethora of Foreign M...</td>\n",
              "      <td>We need to exclude from that list the late Mr ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>train_zero_shot.EN.168.4</td>\n",
              "      <td>EN</td>\n",
              "      <td>double dutch</td>\n",
              "      <td>zero_shot</td>\n",
              "      <td>While pharmaceutical companies were researchin...</td>\n",
              "      <td>Turns out that these people were speaking doub...</td>\n",
              "      <td>So why aren’t Big Macs sold all over the world...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>train_zero_shot.EN.168.5</td>\n",
              "      <td>EN</td>\n",
              "      <td>double dutch</td>\n",
              "      <td>zero_shot</td>\n",
              "      <td>Coronavirus in Europe   * Brexit   * Brussels ...</td>\n",
              "      <td>Is Flemish premier talking double Dutch?</td>\n",
              "      <td>Three months before the Belgians take over the...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4486</th>\n",
              "      <td>train_zero_shot.PT.351.8</td>\n",
              "      <td>PT</td>\n",
              "      <td>gato-pingado</td>\n",
              "      <td>zero_shot</td>\n",
              "      <td>Era o ano de 1968.</td>\n",
              "      <td>A estação de passageiros tranquila, com um ou...</td>\n",
              "      <td>Com os primeiros raios solares, o potente DC 8...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4487</th>\n",
              "      <td>train_zero_shot.PT.351.9</td>\n",
              "      <td>PT</td>\n",
              "      <td>gato-pingado</td>\n",
              "      <td>zero_shot</td>\n",
              "      <td>De acordo com informações vazadas de dentro da...</td>\n",
              "      <td>Segundo a fonte, o programa vai seguir um form...</td>\n",
              "      <td>Leonardo fica inchado, perde a voz e vício mor...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4488</th>\n",
              "      <td>train_zero_shot.PT.351.10</td>\n",
              "      <td>PT</td>\n",
              "      <td>gato-pingado</td>\n",
              "      <td>zero_shot</td>\n",
              "      <td>O radialista Roberto Toledo apresentou o show ...</td>\n",
              "      <td>\"Eu fiquei sentado no chão e realmente só tinh...</td>\n",
              "      <td>Entre os poucos espectadores estava o advogado...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4489</th>\n",
              "      <td>train_zero_shot.PT.351.11</td>\n",
              "      <td>PT</td>\n",
              "      <td>gato-pingado</td>\n",
              "      <td>zero_shot</td>\n",
              "      <td>O carnaval em Goiânia também tem espaço para o...</td>\n",
              "      <td>É o caso do bloco “Gato Pingado”, que reuniu c...</td>\n",
              "      <td>Segundo os organizadores, quando criaram o blo...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4490</th>\n",
              "      <td>train_zero_shot.PT.351.12</td>\n",
              "      <td>PT</td>\n",
              "      <td>gato-pingado</td>\n",
              "      <td>zero_shot</td>\n",
              "      <td>A concentração aconteceu na Avenida Circular, ...</td>\n",
              "      <td>De acordo com presidente do bloco, Paulo de Tá...</td>\n",
              "      <td>“A diferença do bloco é que realmente é um ca...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>4491 rows × 8 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                         DataID  ... Label\n",
              "0      train_zero_shot.EN.168.1  ...     0\n",
              "1      train_zero_shot.EN.168.2  ...     0\n",
              "2      train_zero_shot.EN.168.3  ...     0\n",
              "3      train_zero_shot.EN.168.4  ...     0\n",
              "4      train_zero_shot.EN.168.5  ...     0\n",
              "...                         ...  ...   ...\n",
              "4486   train_zero_shot.PT.351.8  ...     0\n",
              "4487   train_zero_shot.PT.351.9  ...     0\n",
              "4488  train_zero_shot.PT.351.10  ...     0\n",
              "4489  train_zero_shot.PT.351.11  ...     1\n",
              "4490  train_zero_shot.PT.351.12  ...     1\n",
              "\n",
              "[4491 rows x 8 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "outpath = 'Data'\n",
        "    \n",
        "Path( os.path.join( outpath, 'ZeroShot' ) ).mkdir(parents=True, exist_ok=True)\n",
        "Path( os.path.join( outpath, 'OneShot' ) ).mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "create_data( 'SemEval_2022_Task2-idiomaticity/SubTaskA/Data/', outpath )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_PHdYLz9f0mw",
        "outputId": "6f989b61-045b-4cdd-9112-b985458fcc29"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Wrote Data/ZeroShot/train.csv\n",
            "Wrote Data/ZeroShot/dev.csv\n",
            "Wrote Data/ZeroShot/eval.csv\n",
            "Wrote Data/OneShot/train.csv\n",
            "Wrote Data/OneShot/dev.csv\n",
            "Wrote Data/OneShot/eval.csv\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Load Model Here"
      ],
      "metadata": {
        "id": "pYFhmMtuelr3"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qj41wUNP1abs"
      },
      "outputs": [],
      "source": [
        "from transformers import BartForSequenceClassification, BartTokenizer, BartConfig\n",
        "tokenizer = BartTokenizer.from_pretrained('facebook/bart-large-mnli')\n",
        "model = BartForSequenceClassification.from_pretrained('facebook/bart-large-mnli')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "df_eval= pd.read_csv(\"Data/ZeroShot/dev.csv\")"
      ],
      "metadata": {
        "id": "etjK6Ya4lbyW"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_eval.sentence1[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "id": "70CXd7Z4lkbi",
        "outputId": "73fe7f5c-917d-4b8b-d977-84d0aec5bf2d"
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'Does the plumbing predictably rebel, creating a 1,000-foot cascade inside the central utility shaft? Are these interruptions of the good life a necessary condition of the high life? Not at all , says James von Klemperer, president of the architecture firm Kohn Pedersen Fox, which plants skyscrapers all over the world: “In a building that thin, this kind of thing can happen, but it shouldn’t.'"
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.to(torch.device('cuda:0'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q7P98BMXqfXa",
        "outputId": "05c17a93-46d8-4166-c686-dc45def97ecc"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BartForSequenceClassification(\n",
              "  (model): BartModel(\n",
              "    (shared): Embedding(50265, 1024, padding_idx=1)\n",
              "    (encoder): BartEncoder(\n",
              "      (embed_tokens): Embedding(50265, 1024, padding_idx=1)\n",
              "      (embed_positions): BartLearnedPositionalEmbedding(1026, 1024)\n",
              "      (layers): ModuleList(\n",
              "        (0): BartEncoderLayer(\n",
              "          (self_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "        (1): BartEncoderLayer(\n",
              "          (self_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "        (2): BartEncoderLayer(\n",
              "          (self_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "        (3): BartEncoderLayer(\n",
              "          (self_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "        (4): BartEncoderLayer(\n",
              "          (self_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "        (5): BartEncoderLayer(\n",
              "          (self_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "        (6): BartEncoderLayer(\n",
              "          (self_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "        (7): BartEncoderLayer(\n",
              "          (self_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "        (8): BartEncoderLayer(\n",
              "          (self_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "        (9): BartEncoderLayer(\n",
              "          (self_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "        (10): BartEncoderLayer(\n",
              "          (self_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "        (11): BartEncoderLayer(\n",
              "          (self_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "      )\n",
              "      (layernorm_embedding): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "    )\n",
              "    (decoder): BartDecoder(\n",
              "      (embed_tokens): Embedding(50265, 1024, padding_idx=1)\n",
              "      (embed_positions): BartLearnedPositionalEmbedding(1026, 1024)\n",
              "      (layers): ModuleList(\n",
              "        (0): BartDecoderLayer(\n",
              "          (self_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (encoder_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (encoder_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "        (1): BartDecoderLayer(\n",
              "          (self_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (encoder_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (encoder_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "        (2): BartDecoderLayer(\n",
              "          (self_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (encoder_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (encoder_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "        (3): BartDecoderLayer(\n",
              "          (self_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (encoder_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (encoder_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "        (4): BartDecoderLayer(\n",
              "          (self_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (encoder_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (encoder_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "        (5): BartDecoderLayer(\n",
              "          (self_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (encoder_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (encoder_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "        (6): BartDecoderLayer(\n",
              "          (self_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (encoder_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (encoder_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "        (7): BartDecoderLayer(\n",
              "          (self_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (encoder_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (encoder_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "        (8): BartDecoderLayer(\n",
              "          (self_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (encoder_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (encoder_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "        (9): BartDecoderLayer(\n",
              "          (self_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (encoder_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (encoder_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "        (10): BartDecoderLayer(\n",
              "          (self_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (encoder_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (encoder_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "        (11): BartDecoderLayer(\n",
              "          (self_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (encoder_attn): BartAttention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (encoder_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "      )\n",
              "      (layernorm_embedding): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "    )\n",
              "  )\n",
              "  (classification_head): BartClassificationHead(\n",
              "    (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "    (dropout): Dropout(p=0.0, inplace=False)\n",
              "    (out_proj): Linear(in_features=1024, out_features=3, bias=True)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Hypothesis Based Approach"
      ],
      "metadata": {
        "id": "kmBf5TReuagX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import classification_report\n",
        "import numpy as np\n",
        "import torch\n",
        "def find_class_idiom(mnli_model, mnli_tokenizer, labels, list_of_text):\n",
        "  hypothesis = 'idiomatic'\n",
        "  preds = []\n",
        "\n",
        "  for premise in list_of_text:\n",
        "    input_ids = mnli_tokenizer.encode(premise, hypothesis, return_tensors='pt').to(torch.device('cuda:0'))\n",
        "    logits = mnli_model(input_ids)[0]\n",
        "    entail_contradiction_logits = logits[:,[0, 2]]\n",
        "    probs = entail_contradiction_logits.softmax(dim=1)\n",
        "    \n",
        "    true_prob = probs[:,1].item() * 100\n",
        "    if (true_prob >= 50):\n",
        "      preds.append(0)\n",
        "    else:\n",
        "      preds.append(1)\n",
        "    print(f'Probability that the label is true: {true_prob:0.2f}%')\n",
        "  target_names = ['Idiomatic', 'Non-Idiomatic']\n",
        "  print(classification_report(labels, preds, target_names=target_names))"
      ],
      "metadata": {
        "id": "DER_EyxNlumf"
      },
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "find_class_idiom(model, tokenizer, df_eval.label, df_eval.sentence1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9gK3FoVTnC92",
        "outputId": "f27298a5-c47f-4a2f-e5b4-ece8d5c70e94"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Probability that the label is true: 4.04%\n",
            "Probability that the label is true: 43.44%\n",
            "Probability that the label is true: 18.06%\n",
            "Probability that the label is true: 27.10%\n",
            "Probability that the label is true: 31.84%\n",
            "Probability that the label is true: 81.39%\n",
            "Probability that the label is true: 58.86%\n",
            "Probability that the label is true: 60.69%\n",
            "Probability that the label is true: 74.56%\n",
            "Probability that the label is true: 58.24%\n",
            "Probability that the label is true: 34.85%\n",
            "Probability that the label is true: 3.87%\n",
            "Probability that the label is true: 5.91%\n",
            "Probability that the label is true: 35.79%\n",
            "Probability that the label is true: 34.03%\n",
            "Probability that the label is true: 24.10%\n",
            "Probability that the label is true: 46.77%\n",
            "Probability that the label is true: 48.66%\n",
            "Probability that the label is true: 39.76%\n",
            "Probability that the label is true: 24.58%\n",
            "Probability that the label is true: 26.59%\n",
            "Probability that the label is true: 34.18%\n",
            "Probability that the label is true: 35.32%\n",
            "Probability that the label is true: 26.40%\n",
            "Probability that the label is true: 10.20%\n",
            "Probability that the label is true: 12.48%\n",
            "Probability that the label is true: 40.52%\n",
            "Probability that the label is true: 28.08%\n",
            "Probability that the label is true: 27.19%\n",
            "Probability that the label is true: 26.94%\n",
            "Probability that the label is true: 43.20%\n",
            "Probability that the label is true: 27.52%\n",
            "Probability that the label is true: 14.55%\n",
            "Probability that the label is true: 33.22%\n",
            "Probability that the label is true: 32.38%\n",
            "Probability that the label is true: 23.61%\n",
            "Probability that the label is true: 66.15%\n",
            "Probability that the label is true: 5.11%\n",
            "Probability that the label is true: 41.09%\n",
            "Probability that the label is true: 29.83%\n",
            "Probability that the label is true: 43.46%\n",
            "Probability that the label is true: 18.13%\n",
            "Probability that the label is true: 41.10%\n",
            "Probability that the label is true: 9.04%\n",
            "Probability that the label is true: 23.05%\n",
            "Probability that the label is true: 20.92%\n",
            "Probability that the label is true: 24.66%\n",
            "Probability that the label is true: 27.58%\n",
            "Probability that the label is true: 39.79%\n",
            "Probability that the label is true: 40.43%\n",
            "Probability that the label is true: 43.59%\n",
            "Probability that the label is true: 54.84%\n",
            "Probability that the label is true: 25.06%\n",
            "Probability that the label is true: 35.42%\n",
            "Probability that the label is true: 18.27%\n",
            "Probability that the label is true: 20.19%\n",
            "Probability that the label is true: 53.53%\n",
            "Probability that the label is true: 13.59%\n",
            "Probability that the label is true: 5.36%\n",
            "Probability that the label is true: 29.21%\n",
            "Probability that the label is true: 47.11%\n",
            "Probability that the label is true: 21.32%\n",
            "Probability that the label is true: 17.92%\n",
            "Probability that the label is true: 30.90%\n",
            "Probability that the label is true: 51.51%\n",
            "Probability that the label is true: 13.77%\n",
            "Probability that the label is true: 58.52%\n",
            "Probability that the label is true: 33.51%\n",
            "Probability that the label is true: 31.16%\n",
            "Probability that the label is true: 47.34%\n",
            "Probability that the label is true: 48.68%\n",
            "Probability that the label is true: 39.97%\n",
            "Probability that the label is true: 62.03%\n",
            "Probability that the label is true: 75.64%\n",
            "Probability that the label is true: 34.30%\n",
            "Probability that the label is true: 52.32%\n",
            "Probability that the label is true: 38.06%\n",
            "Probability that the label is true: 51.19%\n",
            "Probability that the label is true: 62.53%\n",
            "Probability that the label is true: 46.62%\n",
            "Probability that the label is true: 44.77%\n",
            "Probability that the label is true: 37.73%\n",
            "Probability that the label is true: 24.45%\n",
            "Probability that the label is true: 34.50%\n",
            "Probability that the label is true: 2.72%\n",
            "Probability that the label is true: 25.08%\n",
            "Probability that the label is true: 21.93%\n",
            "Probability that the label is true: 14.35%\n",
            "Probability that the label is true: 57.89%\n",
            "Probability that the label is true: 30.38%\n",
            "Probability that the label is true: 3.54%\n",
            "Probability that the label is true: 15.53%\n",
            "Probability that the label is true: 12.50%\n",
            "Probability that the label is true: 31.02%\n",
            "Probability that the label is true: 5.08%\n",
            "Probability that the label is true: 16.98%\n",
            "Probability that the label is true: 26.60%\n",
            "Probability that the label is true: 6.22%\n",
            "Probability that the label is true: 30.90%\n",
            "Probability that the label is true: 24.37%\n",
            "Probability that the label is true: 34.19%\n",
            "Probability that the label is true: 15.34%\n",
            "Probability that the label is true: 2.01%\n",
            "Probability that the label is true: 19.49%\n",
            "Probability that the label is true: 19.86%\n",
            "Probability that the label is true: 7.76%\n",
            "Probability that the label is true: 53.72%\n",
            "Probability that the label is true: 14.97%\n",
            "Probability that the label is true: 32.73%\n",
            "Probability that the label is true: 8.86%\n",
            "Probability that the label is true: 37.64%\n",
            "Probability that the label is true: 42.34%\n",
            "Probability that the label is true: 38.49%\n",
            "Probability that the label is true: 98.51%\n",
            "Probability that the label is true: 50.87%\n",
            "Probability that the label is true: 32.66%\n",
            "Probability that the label is true: 29.95%\n",
            "Probability that the label is true: 37.49%\n",
            "Probability that the label is true: 16.56%\n",
            "Probability that the label is true: 48.27%\n",
            "Probability that the label is true: 57.46%\n",
            "Probability that the label is true: 12.50%\n",
            "Probability that the label is true: 64.00%\n",
            "Probability that the label is true: 42.96%\n",
            "Probability that the label is true: 53.53%\n",
            "Probability that the label is true: 51.23%\n",
            "Probability that the label is true: 27.39%\n",
            "Probability that the label is true: 49.59%\n",
            "Probability that the label is true: 12.26%\n",
            "Probability that the label is true: 35.57%\n",
            "Probability that the label is true: 27.76%\n",
            "Probability that the label is true: 22.42%\n",
            "Probability that the label is true: 15.60%\n",
            "Probability that the label is true: 48.40%\n",
            "Probability that the label is true: 18.55%\n",
            "Probability that the label is true: 21.02%\n",
            "Probability that the label is true: 18.75%\n",
            "Probability that the label is true: 5.95%\n",
            "Probability that the label is true: 15.79%\n",
            "Probability that the label is true: 32.59%\n",
            "Probability that the label is true: 2.09%\n",
            "Probability that the label is true: 27.36%\n",
            "Probability that the label is true: 57.21%\n",
            "Probability that the label is true: 47.25%\n",
            "Probability that the label is true: 27.25%\n",
            "Probability that the label is true: 75.66%\n",
            "Probability that the label is true: 28.19%\n",
            "Probability that the label is true: 26.43%\n",
            "Probability that the label is true: 25.32%\n",
            "Probability that the label is true: 9.04%\n",
            "Probability that the label is true: 2.22%\n",
            "Probability that the label is true: 36.34%\n",
            "Probability that the label is true: 28.32%\n",
            "Probability that the label is true: 30.40%\n",
            "Probability that the label is true: 17.60%\n",
            "Probability that the label is true: 1.77%\n",
            "Probability that the label is true: 21.69%\n",
            "Probability that the label is true: 36.73%\n",
            "Probability that the label is true: 50.90%\n",
            "Probability that the label is true: 54.75%\n",
            "Probability that the label is true: 16.13%\n",
            "Probability that the label is true: 53.32%\n",
            "Probability that the label is true: 45.97%\n",
            "Probability that the label is true: 24.30%\n",
            "Probability that the label is true: 46.81%\n",
            "Probability that the label is true: 34.74%\n",
            "Probability that the label is true: 40.48%\n",
            "Probability that the label is true: 25.07%\n",
            "Probability that the label is true: 43.24%\n",
            "Probability that the label is true: 42.82%\n",
            "Probability that the label is true: 19.89%\n",
            "Probability that the label is true: 21.73%\n",
            "Probability that the label is true: 36.10%\n",
            "Probability that the label is true: 8.45%\n",
            "Probability that the label is true: 21.47%\n",
            "Probability that the label is true: 71.68%\n",
            "Probability that the label is true: 5.29%\n",
            "Probability that the label is true: 51.13%\n",
            "Probability that the label is true: 34.49%\n",
            "Probability that the label is true: 44.84%\n",
            "Probability that the label is true: 15.98%\n",
            "Probability that the label is true: 21.25%\n",
            "Probability that the label is true: 7.59%\n",
            "Probability that the label is true: 64.99%\n",
            "Probability that the label is true: 10.53%\n",
            "Probability that the label is true: 33.93%\n",
            "Probability that the label is true: 22.69%\n",
            "Probability that the label is true: 34.95%\n",
            "Probability that the label is true: 31.16%\n",
            "Probability that the label is true: 17.76%\n",
            "Probability that the label is true: 16.14%\n",
            "Probability that the label is true: 53.60%\n",
            "Probability that the label is true: 53.62%\n",
            "Probability that the label is true: 43.99%\n",
            "Probability that the label is true: 41.29%\n",
            "Probability that the label is true: 14.14%\n",
            "Probability that the label is true: 46.87%\n",
            "Probability that the label is true: 58.85%\n",
            "Probability that the label is true: 11.80%\n",
            "Probability that the label is true: 54.90%\n",
            "Probability that the label is true: 28.52%\n",
            "Probability that the label is true: 36.62%\n",
            "Probability that the label is true: 31.04%\n",
            "Probability that the label is true: 41.85%\n",
            "Probability that the label is true: 14.74%\n",
            "Probability that the label is true: 2.13%\n",
            "Probability that the label is true: 57.68%\n",
            "Probability that the label is true: 4.74%\n",
            "Probability that the label is true: 2.17%\n",
            "Probability that the label is true: 9.59%\n",
            "Probability that the label is true: 3.23%\n",
            "Probability that the label is true: 29.51%\n",
            "Probability that the label is true: 49.31%\n",
            "Probability that the label is true: 19.09%\n",
            "Probability that the label is true: 65.04%\n",
            "Probability that the label is true: 83.78%\n",
            "Probability that the label is true: 68.79%\n",
            "Probability that the label is true: 25.86%\n",
            "Probability that the label is true: 24.70%\n",
            "Probability that the label is true: 31.71%\n",
            "Probability that the label is true: 35.09%\n",
            "Probability that the label is true: 43.79%\n",
            "Probability that the label is true: 23.06%\n",
            "Probability that the label is true: 46.90%\n",
            "Probability that the label is true: 35.40%\n",
            "Probability that the label is true: 4.46%\n",
            "Probability that the label is true: 28.22%\n",
            "Probability that the label is true: 26.19%\n",
            "Probability that the label is true: 3.76%\n",
            "Probability that the label is true: 78.27%\n",
            "Probability that the label is true: 16.57%\n",
            "Probability that the label is true: 42.93%\n",
            "Probability that the label is true: 63.70%\n",
            "Probability that the label is true: 24.48%\n",
            "Probability that the label is true: 21.05%\n",
            "Probability that the label is true: 31.88%\n",
            "Probability that the label is true: 44.42%\n",
            "Probability that the label is true: 4.72%\n",
            "Probability that the label is true: 33.59%\n",
            "Probability that the label is true: 25.63%\n",
            "Probability that the label is true: 30.42%\n",
            "Probability that the label is true: 8.59%\n",
            "Probability that the label is true: 43.46%\n",
            "Probability that the label is true: 18.79%\n",
            "Probability that the label is true: 36.57%\n",
            "Probability that the label is true: 59.31%\n",
            "Probability that the label is true: 47.73%\n",
            "Probability that the label is true: 63.32%\n",
            "Probability that the label is true: 70.64%\n",
            "Probability that the label is true: 65.54%\n",
            "Probability that the label is true: 69.67%\n",
            "Probability that the label is true: 83.81%\n",
            "Probability that the label is true: 81.68%\n",
            "Probability that the label is true: 31.48%\n",
            "Probability that the label is true: 69.69%\n",
            "Probability that the label is true: 48.63%\n",
            "Probability that the label is true: 71.39%\n",
            "Probability that the label is true: 15.49%\n",
            "Probability that the label is true: 39.16%\n",
            "Probability that the label is true: 27.55%\n",
            "Probability that the label is true: 74.30%\n",
            "Probability that the label is true: 14.86%\n",
            "Probability that the label is true: 18.83%\n",
            "Probability that the label is true: 34.09%\n",
            "Probability that the label is true: 43.67%\n",
            "Probability that the label is true: 23.72%\n",
            "Probability that the label is true: 80.71%\n",
            "Probability that the label is true: 52.34%\n",
            "Probability that the label is true: 17.65%\n",
            "Probability that the label is true: 44.25%\n",
            "Probability that the label is true: 8.54%\n",
            "Probability that the label is true: 86.60%\n",
            "Probability that the label is true: 48.00%\n",
            "Probability that the label is true: 27.50%\n",
            "Probability that the label is true: 56.66%\n",
            "Probability that the label is true: 65.37%\n",
            "Probability that the label is true: 22.13%\n",
            "Probability that the label is true: 33.52%\n",
            "Probability that the label is true: 8.04%\n",
            "Probability that the label is true: 21.63%\n",
            "Probability that the label is true: 17.45%\n",
            "Probability that the label is true: 25.69%\n",
            "Probability that the label is true: 20.05%\n",
            "Probability that the label is true: 6.66%\n",
            "Probability that the label is true: 16.81%\n",
            "Probability that the label is true: 80.82%\n",
            "Probability that the label is true: 30.10%\n",
            "Probability that the label is true: 52.19%\n",
            "Probability that the label is true: 24.37%\n",
            "Probability that the label is true: 2.25%\n",
            "Probability that the label is true: 18.16%\n",
            "Probability that the label is true: 20.16%\n",
            "Probability that the label is true: 12.51%\n",
            "Probability that the label is true: 36.05%\n",
            "Probability that the label is true: 42.95%\n",
            "Probability that the label is true: 8.48%\n",
            "Probability that the label is true: 53.73%\n",
            "Probability that the label is true: 48.05%\n",
            "Probability that the label is true: 36.01%\n",
            "Probability that the label is true: 17.15%\n",
            "Probability that the label is true: 13.97%\n",
            "Probability that the label is true: 12.32%\n",
            "Probability that the label is true: 59.70%\n",
            "Probability that the label is true: 22.48%\n",
            "Probability that the label is true: 61.34%\n",
            "Probability that the label is true: 68.85%\n",
            "Probability that the label is true: 27.15%\n",
            "Probability that the label is true: 58.02%\n",
            "Probability that the label is true: 38.75%\n",
            "Probability that the label is true: 28.29%\n",
            "Probability that the label is true: 20.38%\n",
            "Probability that the label is true: 65.98%\n",
            "Probability that the label is true: 57.22%\n",
            "Probability that the label is true: 22.98%\n",
            "Probability that the label is true: 58.86%\n",
            "Probability that the label is true: 46.65%\n",
            "Probability that the label is true: 42.28%\n",
            "Probability that the label is true: 58.84%\n",
            "Probability that the label is true: 9.45%\n",
            "Probability that the label is true: 33.37%\n",
            "Probability that the label is true: 69.68%\n",
            "Probability that the label is true: 17.59%\n",
            "Probability that the label is true: 31.46%\n",
            "Probability that the label is true: 36.31%\n",
            "Probability that the label is true: 27.09%\n",
            "Probability that the label is true: 16.87%\n",
            "Probability that the label is true: 22.19%\n",
            "Probability that the label is true: 61.24%\n",
            "Probability that the label is true: 28.17%\n",
            "Probability that the label is true: 13.63%\n",
            "Probability that the label is true: 3.69%\n",
            "Probability that the label is true: 9.44%\n",
            "Probability that the label is true: 70.49%\n",
            "Probability that the label is true: 17.44%\n",
            "Probability that the label is true: 24.35%\n",
            "Probability that the label is true: 13.47%\n",
            "Probability that the label is true: 47.46%\n",
            "Probability that the label is true: 30.33%\n",
            "Probability that the label is true: 34.65%\n",
            "Probability that the label is true: 12.88%\n",
            "Probability that the label is true: 26.32%\n",
            "Probability that the label is true: 22.62%\n",
            "Probability that the label is true: 10.23%\n",
            "Probability that the label is true: 27.03%\n",
            "Probability that the label is true: 24.32%\n",
            "Probability that the label is true: 19.77%\n",
            "Probability that the label is true: 35.37%\n",
            "Probability that the label is true: 32.07%\n",
            "Probability that the label is true: 13.75%\n",
            "Probability that the label is true: 41.69%\n",
            "Probability that the label is true: 46.03%\n",
            "Probability that the label is true: 38.41%\n",
            "Probability that the label is true: 47.17%\n",
            "Probability that the label is true: 5.97%\n",
            "Probability that the label is true: 11.81%\n",
            "Probability that the label is true: 39.53%\n",
            "Probability that the label is true: 19.79%\n",
            "Probability that the label is true: 10.42%\n",
            "Probability that the label is true: 11.97%\n",
            "Probability that the label is true: 10.13%\n",
            "Probability that the label is true: 17.44%\n",
            "Probability that the label is true: 14.11%\n",
            "Probability that the label is true: 40.12%\n",
            "Probability that the label is true: 28.83%\n",
            "Probability that the label is true: 52.23%\n",
            "Probability that the label is true: 25.19%\n",
            "Probability that the label is true: 8.33%\n",
            "Probability that the label is true: 0.92%\n",
            "Probability that the label is true: 22.92%\n",
            "Probability that the label is true: 45.16%\n",
            "Probability that the label is true: 28.47%\n",
            "Probability that the label is true: 30.31%\n",
            "Probability that the label is true: 9.84%\n",
            "Probability that the label is true: 37.57%\n",
            "Probability that the label is true: 33.58%\n",
            "Probability that the label is true: 26.10%\n",
            "Probability that the label is true: 34.01%\n",
            "Probability that the label is true: 33.65%\n",
            "Probability that the label is true: 42.03%\n",
            "Probability that the label is true: 24.17%\n",
            "Probability that the label is true: 37.53%\n",
            "Probability that the label is true: 37.60%\n",
            "Probability that the label is true: 24.66%\n",
            "Probability that the label is true: 32.21%\n",
            "Probability that the label is true: 14.83%\n",
            "Probability that the label is true: 24.33%\n",
            "Probability that the label is true: 64.86%\n",
            "Probability that the label is true: 9.70%\n",
            "Probability that the label is true: 16.18%\n",
            "Probability that the label is true: 31.70%\n",
            "Probability that the label is true: 15.26%\n",
            "Probability that the label is true: 33.47%\n",
            "Probability that the label is true: 22.23%\n",
            "Probability that the label is true: 35.61%\n",
            "Probability that the label is true: 8.74%\n",
            "Probability that the label is true: 6.36%\n",
            "Probability that the label is true: 39.36%\n",
            "Probability that the label is true: 51.27%\n",
            "Probability that the label is true: 48.04%\n",
            "Probability that the label is true: 33.58%\n",
            "Probability that the label is true: 9.85%\n",
            "Probability that the label is true: 40.31%\n",
            "Probability that the label is true: 14.47%\n",
            "Probability that the label is true: 6.45%\n",
            "Probability that the label is true: 26.22%\n",
            "Probability that the label is true: 34.02%\n",
            "Probability that the label is true: 84.12%\n",
            "Probability that the label is true: 61.19%\n",
            "Probability that the label is true: 10.48%\n",
            "Probability that the label is true: 45.13%\n",
            "Probability that the label is true: 89.83%\n",
            "Probability that the label is true: 65.07%\n",
            "Probability that the label is true: 47.62%\n",
            "Probability that the label is true: 48.69%\n",
            "Probability that the label is true: 34.20%\n",
            "Probability that the label is true: 24.77%\n",
            "Probability that the label is true: 18.33%\n",
            "Probability that the label is true: 18.02%\n",
            "Probability that the label is true: 23.72%\n",
            "Probability that the label is true: 21.23%\n",
            "Probability that the label is true: 50.00%\n",
            "Probability that the label is true: 39.90%\n",
            "Probability that the label is true: 32.23%\n",
            "Probability that the label is true: 32.82%\n",
            "Probability that the label is true: 13.41%\n",
            "Probability that the label is true: 29.93%\n",
            "Probability that the label is true: 15.55%\n",
            "Probability that the label is true: 38.65%\n",
            "Probability that the label is true: 17.87%\n",
            "Probability that the label is true: 17.08%\n",
            "Probability that the label is true: 58.18%\n",
            "Probability that the label is true: 41.07%\n",
            "Probability that the label is true: 20.31%\n",
            "Probability that the label is true: 30.94%\n",
            "Probability that the label is true: 17.02%\n",
            "Probability that the label is true: 39.02%\n",
            "Probability that the label is true: 35.82%\n",
            "Probability that the label is true: 24.49%\n",
            "Probability that the label is true: 3.08%\n",
            "Probability that the label is true: 12.30%\n",
            "Probability that the label is true: 17.95%\n",
            "Probability that the label is true: 40.02%\n",
            "Probability that the label is true: 53.05%\n",
            "Probability that the label is true: 33.17%\n",
            "Probability that the label is true: 27.56%\n",
            "Probability that the label is true: 28.53%\n",
            "Probability that the label is true: 50.82%\n",
            "Probability that the label is true: 25.63%\n",
            "Probability that the label is true: 0.04%\n",
            "Probability that the label is true: 4.26%\n",
            "Probability that the label is true: 33.00%\n",
            "Probability that the label is true: 22.61%\n",
            "Probability that the label is true: 57.72%\n",
            "Probability that the label is true: 57.59%\n",
            "Probability that the label is true: 58.13%\n",
            "Probability that the label is true: 23.02%\n",
            "Probability that the label is true: 37.40%\n",
            "Probability that the label is true: 21.15%\n",
            "Probability that the label is true: 34.29%\n",
            "Probability that the label is true: 33.08%\n",
            "Probability that the label is true: 27.22%\n",
            "Probability that the label is true: 22.27%\n",
            "Probability that the label is true: 34.43%\n",
            "Probability that the label is true: 57.88%\n",
            "Probability that the label is true: 47.54%\n",
            "Probability that the label is true: 9.63%\n",
            "Probability that the label is true: 80.31%\n",
            "Probability that the label is true: 55.83%\n",
            "Probability that the label is true: 70.31%\n",
            "Probability that the label is true: 63.85%\n",
            "Probability that the label is true: 84.15%\n",
            "Probability that the label is true: 53.38%\n",
            "Probability that the label is true: 58.92%\n",
            "Probability that the label is true: 50.91%\n",
            "Probability that the label is true: 59.90%\n",
            "Probability that the label is true: 43.21%\n",
            "Probability that the label is true: 42.64%\n",
            "Probability that the label is true: 66.65%\n",
            "Probability that the label is true: 75.13%\n",
            "Probability that the label is true: 72.82%\n",
            "Probability that the label is true: 86.38%\n",
            "Probability that the label is true: 85.09%\n",
            "Probability that the label is true: 55.42%\n",
            "Probability that the label is true: 79.48%\n",
            "Probability that the label is true: 67.27%\n",
            "Probability that the label is true: 63.61%\n",
            "Probability that the label is true: 74.16%\n",
            "Probability that the label is true: 67.41%\n",
            "Probability that the label is true: 84.76%\n",
            "Probability that the label is true: 60.08%\n",
            "Probability that the label is true: 42.38%\n",
            "Probability that the label is true: 78.97%\n",
            "Probability that the label is true: 70.28%\n",
            "Probability that the label is true: 79.57%\n",
            "Probability that the label is true: 64.34%\n",
            "Probability that the label is true: 31.16%\n",
            "Probability that the label is true: 71.08%\n",
            "Probability that the label is true: 39.31%\n",
            "Probability that the label is true: 44.44%\n",
            "Probability that the label is true: 56.63%\n",
            "Probability that the label is true: 54.65%\n",
            "Probability that the label is true: 56.37%\n",
            "Probability that the label is true: 41.81%\n",
            "Probability that the label is true: 61.37%\n",
            "Probability that the label is true: 49.13%\n",
            "Probability that the label is true: 83.68%\n",
            "Probability that the label is true: 69.06%\n",
            "Probability that the label is true: 49.25%\n",
            "Probability that the label is true: 66.58%\n",
            "Probability that the label is true: 40.88%\n",
            "Probability that the label is true: 46.92%\n",
            "Probability that the label is true: 69.39%\n",
            "Probability that the label is true: 59.57%\n",
            "Probability that the label is true: 52.52%\n",
            "Probability that the label is true: 60.43%\n",
            "Probability that the label is true: 81.56%\n",
            "Probability that the label is true: 41.76%\n",
            "Probability that the label is true: 73.12%\n",
            "Probability that the label is true: 67.94%\n",
            "Probability that the label is true: 53.38%\n",
            "Probability that the label is true: 71.34%\n",
            "Probability that the label is true: 68.84%\n",
            "Probability that the label is true: 71.88%\n",
            "Probability that the label is true: 63.82%\n",
            "Probability that the label is true: 61.32%\n",
            "Probability that the label is true: 74.56%\n",
            "Probability that the label is true: 94.44%\n",
            "Probability that the label is true: 55.62%\n",
            "Probability that the label is true: 73.37%\n",
            "Probability that the label is true: 56.44%\n",
            "Probability that the label is true: 45.81%\n",
            "Probability that the label is true: 79.15%\n",
            "Probability that the label is true: 86.45%\n",
            "Probability that the label is true: 77.74%\n",
            "Probability that the label is true: 72.58%\n",
            "Probability that the label is true: 60.67%\n",
            "Probability that the label is true: 62.57%\n",
            "Probability that the label is true: 85.56%\n",
            "Probability that the label is true: 61.26%\n",
            "Probability that the label is true: 63.90%\n",
            "Probability that the label is true: 63.40%\n",
            "Probability that the label is true: 79.49%\n",
            "Probability that the label is true: 58.73%\n",
            "Probability that the label is true: 62.56%\n",
            "Probability that the label is true: 78.49%\n",
            "Probability that the label is true: 70.06%\n",
            "Probability that the label is true: 71.20%\n",
            "Probability that the label is true: 60.33%\n",
            "Probability that the label is true: 64.04%\n",
            "Probability that the label is true: 77.82%\n",
            "Probability that the label is true: 60.00%\n",
            "Probability that the label is true: 53.09%\n",
            "Probability that the label is true: 59.40%\n",
            "Probability that the label is true: 72.81%\n",
            "Probability that the label is true: 69.73%\n",
            "Probability that the label is true: 79.40%\n",
            "Probability that the label is true: 28.57%\n",
            "Probability that the label is true: 64.99%\n",
            "Probability that the label is true: 70.64%\n",
            "Probability that the label is true: 66.37%\n",
            "Probability that the label is true: 54.80%\n",
            "Probability that the label is true: 65.67%\n",
            "Probability that the label is true: 83.81%\n",
            "Probability that the label is true: 57.06%\n",
            "Probability that the label is true: 52.39%\n",
            "Probability that the label is true: 51.96%\n",
            "Probability that the label is true: 77.61%\n",
            "Probability that the label is true: 69.14%\n",
            "Probability that the label is true: 84.75%\n",
            "Probability that the label is true: 67.48%\n",
            "Probability that the label is true: 63.31%\n",
            "Probability that the label is true: 45.79%\n",
            "Probability that the label is true: 23.21%\n",
            "Probability that the label is true: 65.39%\n",
            "Probability that the label is true: 3.85%\n",
            "Probability that the label is true: 57.84%\n",
            "Probability that the label is true: 10.84%\n",
            "Probability that the label is true: 15.13%\n",
            "Probability that the label is true: 33.31%\n",
            "Probability that the label is true: 77.14%\n",
            "Probability that the label is true: 74.55%\n",
            "Probability that the label is true: 62.02%\n",
            "Probability that the label is true: 69.18%\n",
            "Probability that the label is true: 71.20%\n",
            "Probability that the label is true: 42.62%\n",
            "Probability that the label is true: 78.59%\n",
            "Probability that the label is true: 62.71%\n",
            "Probability that the label is true: 37.98%\n",
            "Probability that the label is true: 60.54%\n",
            "Probability that the label is true: 71.82%\n",
            "Probability that the label is true: 24.32%\n",
            "Probability that the label is true: 22.88%\n",
            "Probability that the label is true: 67.55%\n",
            "Probability that the label is true: 72.47%\n",
            "Probability that the label is true: 62.90%\n",
            "Probability that the label is true: 50.13%\n",
            "Probability that the label is true: 43.01%\n",
            "Probability that the label is true: 43.14%\n",
            "Probability that the label is true: 37.71%\n",
            "Probability that the label is true: 44.83%\n",
            "Probability that the label is true: 54.71%\n",
            "Probability that the label is true: 75.09%\n",
            "Probability that the label is true: 71.50%\n",
            "Probability that the label is true: 69.69%\n",
            "Probability that the label is true: 77.63%\n",
            "Probability that the label is true: 77.87%\n",
            "Probability that the label is true: 51.74%\n",
            "Probability that the label is true: 46.39%\n",
            "Probability that the label is true: 78.18%\n",
            "Probability that the label is true: 28.87%\n",
            "Probability that the label is true: 50.42%\n",
            "Probability that the label is true: 44.88%\n",
            "Probability that the label is true: 59.65%\n",
            "Probability that the label is true: 41.61%\n",
            "Probability that the label is true: 15.00%\n",
            "Probability that the label is true: 49.99%\n",
            "Probability that the label is true: 15.08%\n",
            "Probability that the label is true: 56.55%\n",
            "Probability that the label is true: 23.50%\n",
            "Probability that the label is true: 51.97%\n",
            "Probability that the label is true: 75.98%\n",
            "Probability that the label is true: 50.09%\n",
            "Probability that the label is true: 61.39%\n",
            "Probability that the label is true: 38.27%\n",
            "Probability that the label is true: 80.03%\n",
            "Probability that the label is true: 49.99%\n",
            "Probability that the label is true: 51.26%\n",
            "Probability that the label is true: 26.33%\n",
            "Probability that the label is true: 70.85%\n",
            "Probability that the label is true: 45.17%\n",
            "Probability that the label is true: 68.69%\n",
            "Probability that the label is true: 81.12%\n",
            "Probability that the label is true: 71.95%\n",
            "Probability that the label is true: 43.20%\n",
            "Probability that the label is true: 55.48%\n",
            "Probability that the label is true: 81.85%\n",
            "Probability that the label is true: 58.39%\n",
            "Probability that the label is true: 64.31%\n",
            "Probability that the label is true: 50.88%\n",
            "Probability that the label is true: 60.38%\n",
            "Probability that the label is true: 58.70%\n",
            "Probability that the label is true: 69.50%\n",
            "Probability that the label is true: 72.16%\n",
            "Probability that the label is true: 62.61%\n",
            "Probability that the label is true: 73.69%\n",
            "Probability that the label is true: 66.75%\n",
            "Probability that the label is true: 52.55%\n",
            "Probability that the label is true: 66.73%\n",
            "Probability that the label is true: 74.24%\n",
            "Probability that the label is true: 68.15%\n",
            "Probability that the label is true: 61.06%\n",
            "Probability that the label is true: 55.69%\n",
            "Probability that the label is true: 64.65%\n",
            "Probability that the label is true: 78.86%\n",
            "Probability that the label is true: 37.95%\n",
            "Probability that the label is true: 48.47%\n",
            "Probability that the label is true: 76.40%\n",
            "Probability that the label is true: 82.27%\n",
            "Probability that the label is true: 19.78%\n",
            "Probability that the label is true: 81.11%\n",
            "Probability that the label is true: 66.68%\n",
            "Probability that the label is true: 38.72%\n",
            "Probability that the label is true: 56.04%\n",
            "Probability that the label is true: 59.91%\n",
            "Probability that the label is true: 56.84%\n",
            "Probability that the label is true: 56.57%\n",
            "Probability that the label is true: 64.56%\n",
            "Probability that the label is true: 50.31%\n",
            "Probability that the label is true: 63.42%\n",
            "Probability that the label is true: 57.93%\n",
            "Probability that the label is true: 77.60%\n",
            "Probability that the label is true: 56.12%\n",
            "Probability that the label is true: 36.39%\n",
            "Probability that the label is true: 60.65%\n",
            "Probability that the label is true: 75.09%\n",
            "Probability that the label is true: 64.48%\n",
            "Probability that the label is true: 83.72%\n",
            "Probability that the label is true: 52.97%\n",
            "Probability that the label is true: 72.41%\n",
            "Probability that the label is true: 53.34%\n",
            "Probability that the label is true: 52.17%\n",
            "Probability that the label is true: 51.73%\n",
            "Probability that the label is true: 65.36%\n",
            "Probability that the label is true: 52.97%\n",
            "Probability that the label is true: 69.14%\n",
            "Probability that the label is true: 71.46%\n",
            "Probability that the label is true: 73.02%\n",
            "Probability that the label is true: 64.09%\n",
            "Probability that the label is true: 76.45%\n",
            "Probability that the label is true: 63.26%\n",
            "Probability that the label is true: 67.14%\n",
            "Probability that the label is true: 70.95%\n",
            "Probability that the label is true: 83.24%\n",
            "Probability that the label is true: 84.34%\n",
            "Probability that the label is true: 43.09%\n",
            "Probability that the label is true: 23.90%\n",
            "Probability that the label is true: 51.61%\n",
            "Probability that the label is true: 77.12%\n",
            "Probability that the label is true: 49.68%\n",
            "Probability that the label is true: 73.70%\n",
            "Probability that the label is true: 76.90%\n",
            "Probability that the label is true: 82.76%\n",
            "Probability that the label is true: 80.03%\n",
            "Probability that the label is true: 68.91%\n",
            "Probability that the label is true: 76.22%\n",
            "Probability that the label is true: 74.98%\n",
            "Probability that the label is true: 54.65%\n",
            "Probability that the label is true: 76.45%\n",
            "Probability that the label is true: 72.61%\n",
            "Probability that the label is true: 60.44%\n",
            "Probability that the label is true: 72.11%\n",
            "Probability that the label is true: 18.28%\n",
            "Probability that the label is true: 69.74%\n",
            "Probability that the label is true: 78.58%\n",
            "Probability that the label is true: 61.03%\n",
            "Probability that the label is true: 76.64%\n",
            "Probability that the label is true: 58.29%\n",
            "Probability that the label is true: 77.12%\n",
            "Probability that the label is true: 61.90%\n",
            "Probability that the label is true: 76.60%\n",
            "Probability that the label is true: 73.33%\n",
            "Probability that the label is true: 62.13%\n",
            "Probability that the label is true: 64.73%\n",
            "Probability that the label is true: 66.39%\n",
            "Probability that the label is true: 83.55%\n",
            "Probability that the label is true: 67.14%\n",
            "Probability that the label is true: 75.28%\n",
            "Probability that the label is true: 71.50%\n",
            "Probability that the label is true: 77.34%\n",
            "Probability that the label is true: 76.50%\n",
            "Probability that the label is true: 78.18%\n",
            "Probability that the label is true: 75.61%\n",
            "Probability that the label is true: 68.58%\n",
            "Probability that the label is true: 51.67%\n",
            "Probability that the label is true: 59.88%\n",
            "Probability that the label is true: 55.09%\n",
            "Probability that the label is true: 65.02%\n",
            "Probability that the label is true: 61.43%\n",
            "Probability that the label is true: 85.96%\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "    Idiomatic       0.47      0.43      0.45       336\n",
            "Non-Idiomatic       0.56      0.60      0.58       403\n",
            "\n",
            "     accuracy                           0.52       739\n",
            "    macro avg       0.51      0.51      0.51       739\n",
            " weighted avg       0.52      0.52      0.52       739\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "_t1piULbe3t8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "Fy7l9yLle5wZ"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}