nlppppppp_(1).ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "id": "8e7ab23e",
      "metadata": {
        "id": "8e7ab23e"
      },
      "source": [
        "# IMPORTING REQUIRED LIBRARIES"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "4ca819ff",
      "metadata": {
        "id": "4ca819ff"
      },
      "outputs": [],
      "source": [
        "from transformers import BertTokenizer, BertModel\n",
        "import torch\n",
        "from sklearn.cluster import KMeans\n",
        "from sklearn.metrics import pairwise_distances_argmin_min\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import re\n",
        "import nltk\n",
        "from collections import Counter\n",
        "from transformers import BertTokenizer, BertModel\n",
        "import torch\n",
        "from sklearn.cluster import KMeans\n",
        "from sklearn.metrics import pairwise_distances_argmin_min\n"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "6aeb90f0",
      "metadata": {
        "id": "6aeb90f0"
      },
      "source": [
        "# READING THE DATA"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "42f66414",
      "metadata": {
        "id": "42f66414",
        "outputId": "00370d4b-b1c5-4010-808c-8a2f64d23206",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                              review sentiment\n",
              "0  One of the other reviewers has mentioned that ...  positive\n",
              "1  A wonderful little production. <br /><br />The...  positive\n",
              "2  I thought this was a wonderful way to spend ti...  positive\n",
              "3  Basically there's a family where a little boy ...  negative\n",
              "4  Petter Mattei's \"Love in the Time of Money\" is...  positive"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a591e21e-b2f4-41bd-aef4-113d0957f3cb\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>review</th>\n",
              "      <th>sentiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>One of the other reviewers has mentioned that ...</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>A wonderful little production. &lt;br /&gt;&lt;br /&gt;The...</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>I thought this was a wonderful way to spend ti...</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Basically there's a family where a little boy ...</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Petter Mattei's \"Love in the Time of Money\" is...</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a591e21e-b2f4-41bd-aef4-113d0957f3cb')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-a591e21e-b2f4-41bd-aef4-113d0957f3cb button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-a591e21e-b2f4-41bd-aef4-113d0957f3cb');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-caa4a443-6ad7-463a-8a0c-cf9ff6e1fdcc\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-caa4a443-6ad7-463a-8a0c-cf9ff6e1fdcc')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-caa4a443-6ad7-463a-8a0c-cf9ff6e1fdcc button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "data",
              "summary": "{\n  \"name\": \"data\",\n  \"rows\": 50000,\n  \"fields\": [\n    {\n      \"column\": \"review\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 49582,\n        \"samples\": [\n          \"\\\"Soul Plane\\\" is a horrible attempt at comedy that only should appeal people with thick skulls, bloodshot eyes and furry pawns. <br /><br />The plot is not only incoherent but also non-existent, acting is mostly sub sub-par with a gang of highly moronic and dreadful characters thrown in for bad measure, jokes are often spotted miles ahead and almost never even a bit amusing. This movie lacks any structure and is full of racial stereotypes that must have seemed old even in the fifties, the only thing it really has going for it is some pretty ladies, but really, if you want that you can rent something from the \\\"Adult\\\" section. OK?<br /><br />I can hardly see anything here to recommend since you'll probably have a lot a better and productive time chasing rats with a sledgehammer or inventing waterproof teabags or whatever.<br /><br />2/10\",\n          \"Guest from the Future tells a fascinating story of time travel, friendship, battle of good and evil -- all with a small budget, child actors, and few special effects. Something for Spielberg and Lucas to learn from. ;) A sixth-grader Kolya \\\"Nick\\\" Gerasimov finds a time machine in the basement of a decrepit building and travels 100 years into the future. He discovers a near-perfect, utopian society where robots play guitars and write poetry, everyone is kind to each other and people enjoy everything technology has to offer. Alice is the daughter of a prominent scientist who invented a device called Mielophone that allows to read minds of humans and animals. The device can be put to both good and bad use, depending on whose hands it falls into. When two evil space pirates from Saturn who want to rule the universe attempt to steal Mielophone, it falls into the hands of 20th century school boy Nick. With the pirates hot on his tracks, he travels back to his time, followed by the pirates, and Alice. Chaos, confusion and funny situations follow as the luckless pirates try to blend in with the earthlings. Alice enrolls in the same school Nick goes to and demonstrates superhuman abilities in PE class. The catch is, Alice doesn't know what Nick looks like, while the pirates do. Also, the pirates are able to change their appearance and turn literally into anyone. (Hmm, I wonder if this is where James Cameron got the idea for Terminator...) Who gets to Nick -- and Mielophone -- first? Excellent plot, non-stop adventures, and great soundtrack. I wish Hollywood made kid movies like this one...\",\n          \"\\\"National Treasure\\\" (2004) is a thoroughly misguided hodge-podge of plot entanglements that borrow from nearly every cloak and dagger government conspiracy clich\\u00e9 that has ever been written. The film stars Nicholas Cage as Benjamin Franklin Gates (how precious is that, I ask you?); a seemingly normal fellow who, for no other reason than being of a lineage of like-minded misguided fortune hunters, decides to steal a 'national treasure' that has been hidden by the United States founding fathers. After a bit of subtext and background that plays laughably (unintentionally) like Indiana Jones meets The Patriot, the film degenerates into one misguided whimsy after another \\u0096 attempting to create a 'Stanley Goodspeed' regurgitation of Nicholas Cage and launch the whole convoluted mess forward with a series of high octane, but disconnected misadventures.<br /><br />The relevancy and logic to having George Washington and his motley crew of patriots burying a king's ransom someplace on native soil, and then, going through the meticulous plan of leaving clues scattered throughout U.S. currency art work, is something that director Jon Turteltaub never quite gets around to explaining. Couldn't Washington found better usage for such wealth during the start up of the country? Hence, we are left with a mystery built on top of an enigma that is already on shaky ground by the time Ben appoints himself the new custodian of this untold wealth. Ben's intentions are noble \\u0096 if confusing. He's set on protecting the treasure. For who and when?\\u0085your guess is as good as mine.<br /><br />But there are a few problems with Ben's crusade. First up, his friend, Ian Holmes (Sean Bean) decides that he can't wait for Ben to make up his mind about stealing the Declaration of Independence from the National Archives (oh, yeah \\u0096 brilliant idea!). Presumably, the back of that famous document holds the secret answer to the ultimate fortune. So Ian tries to kill Ben. The assassination attempt is, of course, unsuccessful, if overly melodramatic. It also affords Ben the opportunity to pick up, and pick on, the very sultry curator of the archives, Abigail Chase (Diane Kruger). She thinks Ben is clearly a nut \\u0096 at least at the beginning. But true to action/romance form, Abby's resolve melts quicker than you can say, \\\"is that the Hope Diamond?\\\" The film moves into full X-File-ish mode, as the FBI, mistakenly believing that Ben is behind the theft, retaliate in various benign ways that lead to a multi-layering of action sequences reminiscent of Mission Impossible meets The Fugitive. Honestly, don't those guys ever get 'intelligence' information that is correct? In the final analysis, \\\"National Treasure\\\" isn't great film making, so much as it's a patchwork rehash of tired old bits from other movies, woven together from scraps, the likes of which would make IL' Betsy Ross blush.<br /><br />The Buena Vista DVD delivers a far more generous treatment than this film is deserving of. The anamorphic widescreen picture exhibits a very smooth and finely detailed image with very rich colors, natural flesh tones, solid blacks and clean whites. The stylized image is also free of blemishes and digital enhancements. The audio is 5.1 and delivers a nice sonic boom to your side and rear speakers with intensity and realism. Extras include a host of promotional junket material that is rather deep and over the top in its explanation of how and why this film was made. If only, as an audience, we had had more clarification as to why Ben and co. were chasing after an illusive treasure, this might have been one good flick. Extras conclude with the theatrical trailer, audio commentary and deleted scenes. Not for the faint-hearted \\u0096 just the thick-headed.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"sentiment\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"negative\",\n          \"positive\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 11
        }
      ],
      "source": [
        "data=pd.read_csv(r\"/content/IMDB Dataset.csv\")\n",
        "data.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "70bd568e",
      "metadata": {
        "id": "70bd568e"
      },
      "source": [
        "# EDA"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "4999831e",
      "metadata": {
        "id": "4999831e",
        "outputId": "940bc7c8-d166-4430-9ca4-eec0c6b2c1cf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                   review sentiment\n",
              "count                                               50000     50000\n",
              "unique                                              49582         2\n",
              "top     Loved today's show!!! It was a variety and not...  positive\n",
              "freq                                                    5     25000"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b6fa7da9-b7ca-4f4f-bad6-5ef319a620d0\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>review</th>\n",
              "      <th>sentiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>50000</td>\n",
              "      <td>50000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>unique</th>\n",
              "      <td>49582</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>top</th>\n",
              "      <td>Loved today's show!!! It was a variety and not...</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>freq</th>\n",
              "      <td>5</td>\n",
              "      <td>25000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b6fa7da9-b7ca-4f4f-bad6-5ef319a620d0')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-b6fa7da9-b7ca-4f4f-bad6-5ef319a620d0 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-b6fa7da9-b7ca-4f4f-bad6-5ef319a620d0');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-59373868-5247-49de-a99b-1fdcac59a4d1\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-59373868-5247-49de-a99b-1fdcac59a4d1')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-59373868-5247-49de-a99b-1fdcac59a4d1 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"data\",\n  \"rows\": 4,\n  \"fields\": [\n    {\n      \"column\": \"review\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 4,\n        \"samples\": [\n          49582,\n          \"5\",\n          \"50000\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"sentiment\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 4,\n        \"samples\": [\n          2,\n          \"25000\",\n          \"50000\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 12
        }
      ],
      "source": [
        "data.describe()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "9be5debf",
      "metadata": {
        "id": "9be5debf",
        "outputId": "5d9e3e58-3ca8-4918-e5b5-635fbbe05f33",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       review  sentiment\n",
              "0       False      False\n",
              "1       False      False\n",
              "2       False      False\n",
              "3       False      False\n",
              "4       False      False\n",
              "...       ...        ...\n",
              "49995   False      False\n",
              "49996   False      False\n",
              "49997   False      False\n",
              "49998   False      False\n",
              "49999   False      False\n",
              "\n",
              "[50000 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d6f93af1-605b-466d-929a-2d713c64f665\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>review</th>\n",
              "      <th>sentiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49995</th>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49996</th>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49997</th>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49998</th>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49999</th>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>50000 rows × 2 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d6f93af1-605b-466d-929a-2d713c64f665')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-d6f93af1-605b-466d-929a-2d713c64f665 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-d6f93af1-605b-466d-929a-2d713c64f665');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-40bec109-fc8b-4d6a-8e9e-1c076a6e8855\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-40bec109-fc8b-4d6a-8e9e-1c076a6e8855')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-40bec109-fc8b-4d6a-8e9e-1c076a6e8855 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"data\",\n  \"rows\": 50000,\n  \"fields\": [\n    {\n      \"column\": \"review\",\n      \"properties\": {\n        \"dtype\": \"boolean\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          false\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"sentiment\",\n      \"properties\": {\n        \"dtype\": \"boolean\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          false\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 13
        }
      ],
      "source": [
        "data.isnull()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "61466187",
      "metadata": {
        "id": "61466187",
        "outputId": "79c91da0-ed27-417e-8ec0-b2e3ccaf031b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 50000 entries, 0 to 49999\n",
            "Data columns (total 2 columns):\n",
            " #   Column     Non-Null Count  Dtype \n",
            "---  ------     --------------  ----- \n",
            " 0   review     50000 non-null  object\n",
            " 1   sentiment  50000 non-null  object\n",
            "dtypes: object(2)\n",
            "memory usage: 781.4+ KB\n"
          ]
        }
      ],
      "source": [
        "data.info()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "8de7c413",
      "metadata": {
        "id": "8de7c413"
      },
      "source": [
        "# PREPROCESSING\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b37c976b",
      "metadata": {
        "id": "b37c976b"
      },
      "outputs": [],
      "source": [
        "def remove_tags(string):\n",
        "    removelist = \"\"\n",
        "    result = re.sub('','',string)          #remove HTML tags\n",
        "    result = re.sub('https://.*','',result)   #remove URLs\n",
        "    result = result.lower()\n",
        "    result = result.replace('<br />', '')\n",
        "    return result\n",
        "data['review']=data['review'].apply(lambda cw : remove_tags(cw))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6c856cd1",
      "metadata": {
        "id": "6c856cd1",
        "outputId": "beb6ab9a-813d-40af-ec68-6ecda3b4f5e8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "32532230"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ],
      "source": [
        "reviews=''.join(data[data['sentiment']=='positive']['review'])\n",
        "len(reviews)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "bd08e3d5",
      "metadata": {
        "id": "bd08e3d5",
        "outputId": "33543f34-d9cb-4936-d409-37c99da65af6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 157
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"one of the other reviewers has mentioned that after watching just 1 oz episode you'll be hooked. they are right, as this is exactly what happened with me.the first thing that struck me about oz was its brutality and unflinching scenes of violence, which set in right from the word go. trust me, this is not a show for the faint hearted or timid. this show pulls no punches with regards to drugs, sex or violence. its is hardcore, in the classic use of the word.it is called oz as that is the nickname given to the oswald maximum security state penitentary. it focuses mainly on emerald city, an experimental section of the prison where all the cells have glass fronts and face inwards, so privacy is not high on the agenda. em city is home to many..aryans, muslims, gangstas, latinos, christians, italians, irish and more....so scuffles, death stares, dodgy dealings and shady agreements are never far away.i would say the main appeal of the show is due to the fact that it goes where other shows wouldn't dare. forget pretty pictures painted for mainstream audiences, forget charm, forget romance...oz doesn't mess around. the first episode i ever saw struck me as so nasty it was surreal, i couldn't say i was ready for it, but as i watched more, i developed a taste for oz, and got accustomed to the high levels of graphic violence. not just violence, but injustice (crooked guards who'll be sold out for a nickel, inmates who'll kill on order and get away with it\""
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 17
        }
      ],
      "source": [
        "reviews[:1467]"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "4cbd3d95",
      "metadata": {
        "id": "4cbd3d95"
      },
      "source": [
        "# SUMMARIZATION"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "a0506bea",
      "metadata": {
        "id": "a0506bea",
        "outputId": "dc095f52-b384-43d9-e3ce-8f1faa1cc940",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Enter your text: Multi-head attention The above visualization shows one attention mechanism within the model. BERT actually learns multiple attention mechanisms, called heads, which operate in parallel to one another. As we’ll see shortly, multi-head attention enables the model to capture a broader range of relationships between words than would be possible with a single attention mechanism.  BERT also stacks multiple layers of attention, each of which operates on the output of the layer that came before. Through this repeated composition of word embeddings, BERT is able to form very rich representations as it gets to the deepest layers of the model.  Because the attention heads do not share parameters, each head learns a unique attention pattern. The version of BERT that we consider here — BERT Base — has 12 layers and 12 heads, resulting in a total of 12 x 12 = 144 distinct attention mechanisms. We can visualize attention in all of the heads at once, using the model view (available in interactive form here):\n"
          ]
        }
      ],
      "source": [
        "# Load pre-trained model tokenizer (vocabulary)\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "\n",
        "# Load pre-trained model (weights)\n",
        "model = BertModel.from_pretrained('bert-base-uncased', output_hidden_states=True)\n",
        "\n",
        "# Put the model in \"evaluation\" mode, meaning feed-forward operation.\n",
        "model.eval()\n",
        "\n",
        "def get_embeddings(text):\n",
        "    # Check if the text is not empty or None\n",
        "    if text:\n",
        "        marked_text = \"[CLS] \" + text + \" [SEP]\"\n",
        "        tokenized_text = tokenizer.tokenize(marked_text)\n",
        "        indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)\n",
        "        segments_ids = [1] * len(tokenized_text)\n",
        "\n",
        "        # Convert inputs to PyTorch tensors\n",
        "        tokens_tensor = torch.tensor([indexed_tokens])\n",
        "        segments_tensors = torch.tensor([segments_ids])\n",
        "\n",
        "        # Run the text through BERT, and collect all of the hidden states produced from all 12 layers.\n",
        "        with torch.no_grad():\n",
        "            outputs = model(tokens_tensor, segments_tensors)\n",
        "            hidden_states = outputs[2]\n",
        "\n",
        "        # token_vecs is a tensor with shape [22 x 768]\n",
        "        token_vecs = hidden_states[-2][0]\n",
        "\n",
        "        # Calculate the average of all 22 token vectors.\n",
        "        sentence_embedding = torch.mean(token_vecs, dim=0)\n",
        "\n",
        "        return sentence_embedding.numpy()\n",
        "    else:\n",
        "        # Return a zero vector for empty sentences\n",
        "        return np.zeros((768,))\n",
        "\n",
        "# Add a count function to count the words in the text\n",
        "def count_words(text):\n",
        "    words = text.split()\n",
        "    return len(words)\n",
        "\n",
        "# Get user-defined input for the text\n",
        "text = input(\"Enter your text: \")\n",
        "\n",
        "def summarize(text):\n",
        "    sentences = text.split('.')\n",
        "    embeddings = [get_embeddings(sentence) for sentence in sentences]\n",
        "    n_clusters = int(np.ceil(len(embeddings) ** 0.5))\n",
        "    kmeans = KMeans(n_clusters=n_clusters)\n",
        "    kmeans = kmeans.fit(embeddings)\n",
        "    avg = []\n",
        "    closest = []\n",
        "    for j in range(n_clusters):\n",
        "        idx = np.where(kmeans.labels_ == j)[0]\n",
        "        avg.append(np.mean(idx))\n",
        "    closest, _ = pairwise_distances_argmin_min(kmeans.cluster_centers_, embeddings)\n",
        "    ordering = sorted(range(n_clusters), key=lambda k: avg[k])\n",
        "    summary = ' '.join([sentences[closest[idx]] for idx in ordering])\n",
        "    return summary\n"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "2f4c08a6",
      "metadata": {
        "id": "2f4c08a6"
      },
      "source": [
        "# Outputs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "bf57bc5e",
      "metadata": {
        "id": "bf57bc5e",
        "outputId": "f3dc56cf-d3f6-4990-8b72-0ac209db1779",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original text:\n",
            "\n",
            "Multi-head attention The above visualization shows one attention mechanism within the model. BERT actually learns multiple attention mechanisms, called heads, which operate in parallel to one another. As we’ll see shortly, multi-head attention enables the model to capture a broader range of relationships between words than would be possible with a single attention mechanism.  BERT also stacks multiple layers of attention, each of which operates on the output of the layer that came before. Through this repeated composition of word embeddings, BERT is able to form very rich representations as it gets to the deepest layers of the model.  Because the attention heads do not share parameters, each head learns a unique attention pattern. The version of BERT that we consider here — BERT Base — has 12 layers and 12 heads, resulting in a total of 12 x 12 = 144 distinct attention mechanisms. We can visualize attention in all of the heads at once, using the model view (available in interactive form here):\n",
            "Word count: 165 \n",
            "\n"
          ]
        }
      ],
      "source": [
        "print(\"Original text:\\n\")\n",
        "print(text)\n",
        "print(\"Word count:\", count_words(text), \"\\n\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "90ba5074",
      "metadata": {
        "id": "90ba5074",
        "outputId": "ce9d4470-4c48-43c8-faf6-c8b280e58c3d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Tokenized text:\n",
            "\n",
            "['multi', '-', 'head', 'attention', 'the', 'above', 'visual', '##ization', 'shows', 'one', 'attention', 'mechanism', 'within', 'the', 'model', '.', 'bert', 'actually', 'learns', 'multiple', 'attention', 'mechanisms', ',', 'called', 'heads', ',', 'which', 'operate', 'in', 'parallel', 'to', 'one', 'another', '.', 'as', 'we', '’', 'll', 'see', 'shortly', ',', 'multi', '-', 'head', 'attention', 'enables', 'the', 'model', 'to', 'capture', 'a', 'broader', 'range', 'of', 'relationships', 'between', 'words', 'than', 'would', 'be', 'possible', 'with', 'a', 'single', 'attention', 'mechanism', '.', 'bert', 'also', 'stacks', 'multiple', 'layers', 'of', 'attention', ',', 'each', 'of', 'which', 'operates', 'on', 'the', 'output', 'of', 'the', 'layer', 'that', 'came', 'before', '.', 'through', 'this', 'repeated', 'composition', 'of', 'word', 'em', '##bed', '##ding', '##s', ',', 'bert', 'is', 'able', 'to', 'form', 'very', 'rich', 'representations', 'as', 'it', 'gets', 'to', 'the', 'deepest', 'layers', 'of', 'the', 'model', '.', 'because', 'the', 'attention', 'heads', 'do', 'not', 'share', 'parameters', ',', 'each', 'head', 'learns', 'a', 'unique', 'attention', 'pattern', '.', 'the', 'version', 'of', 'bert', 'that', 'we', 'consider', 'here', '—', 'bert', 'base', '—', 'has', '12', 'layers', 'and', '12', 'heads', ',', 'resulting', 'in', 'a', 'total', 'of', '12', 'x', '12', '=', '144', 'distinct', 'attention', 'mechanisms', '.', 'we', 'can', 'visual', '##ize', 'attention', 'in', 'all', 'of', 'the', 'heads', 'at', 'once', ',', 'using', 'the', 'model', 'view', '(', 'available', 'in', 'interactive', 'form', 'here', ')', ':']\n",
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# Print the tokenized text\n",
        "print(\"Tokenized text:\\n\")\n",
        "tokenized_text = tokenizer.tokenize(text)\n",
        "print(tokenized_text)\n",
        "print(\"\\n\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c2bae5d3",
      "metadata": {
        "id": "c2bae5d3",
        "outputId": "0e01ec0a-de31-4318-afbe-2a63ebfbf144",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Summary:\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/sklearn/cluster/_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Multi-head attention The above visualization shows one attention mechanism within the model   Because the attention heads do not share parameters, each head learns a unique attention pattern  Through this repeated composition of word embeddings, BERT is able to form very rich representations as it gets to the deepest layers of the model\n",
            "Word count: 52 \n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/sklearn/cluster/_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "print(\"Summary:\\n\")\n",
        "print(summarize(text))\n",
        "print(\"Word count:\", count_words(summarize(text)), \"\\n\")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Web Scrapping"
      ],
      "metadata": {
        "id": "mirPo8Gzu23c"
      },
      "id": "mirPo8Gzu23c"
    },
    {
      "cell_type": "code",
      "source": [
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "import time\n",
        "\n",
        "def scrape_website(url, delay=2):\n",
        "  \"\"\"\n",
        "  Scrapes a website and attempts to capture more content than just the first section/paragraph.\n",
        "\n",
        "  Args:\n",
        "      url: The URL of the website to scrape.\n",
        "      delay: Delay (in seconds) between requests to avoid overloading the server.\n",
        "\n",
        "  Returns:\n",
        "      A string containing the scraped content.\n",
        "  \"\"\"\n",
        "  headers = {'User-Agent': 'Mozilla/5.0'}  # Mimic a browser to avoid blocking\n",
        "  response = requests.get(url, headers=headers)\n",
        "  soup = BeautifulSoup(response.content, 'html.parser')\n",
        "\n",
        "  # Find all paragraphs\n",
        "  paragraphs = soup.find_all('p')\n",
        "\n",
        "  # Combine paragraph text along with headings (optional)\n",
        "  content = \"\"\n",
        "  for element in soup.find_all(['p', 'h1', 'h2', 'h3']):\n",
        "    content += element.text.strip() + \"\\n\"  # Add newline for readability\n",
        "\n",
        "  # Optionally, consider using libraries like Selenium for dynamic content\n",
        "  # or infinite scrolling websites (not covered in this example)\n",
        "\n",
        "  time.sleep(delay)  # Delay to be polite to the server\n",
        "  return content\n",
        "\n",
        "# Example usage\n",
        "url = \"https://www.bbc.com/\"  # Replace with the actual website URL\n",
        "scraped_content = scrape_website(url)\n",
        "print(scraped_content)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QbkE_GJUs-g8",
        "outputId": "3dd85b1b-8245-45c0-9a6c-a258c7c21966"
      },
      "id": "QbkE_GJUs-g8",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "US House approves crucial $61bn war aid for Ukraine\n",
            "The vote will be closely watched in Kyiv, which has warned of an urgent need for more military support.\n",
            "The 'Robin Hood' TikToker taking on landlords\n",
            "Jordan van den Berg's vigilante-style approach has won him fans and critics amid a housing crisis.\n",
            "US House approves crucial $61bn war aid for Ukraine\n",
            "The vote will be closely watched in Kyiv, which has warned of an urgent need for more military support.\n",
            "The 'Robin Hood' TikToker taking on landlords\n",
            "Jordan van den Berg's vigilante-style approach has won him fans and critics amid a housing crisis.\n",
            "US aid could give Ukraine the weapons it needs to slow Russia’s advance\n",
            "The $61bn (£49bn) aid package keeps Ukraine in the fight but it's no silver bullet, BBC's James Waterhouse writes from Kyiv.\n",
            "What the world makes of Trump going on trial\n",
            "Pictures of Donald Trump in a New York courtroom made headlines globally. Here's a flavour of what was said.\n",
            "US migrant crisis shifts from Texas to California border\n",
            "A record surge of illegal border crossings in recent years has made immigration a top voter concern.\n",
            "PM 'appalled' by police treatment of Jewish man\n",
            "A No 10 source says police \"mishandled\" its interaction with a Jewish man at pro-Palestinian protest.\n",
            "Jepchirchir wins in women's only world record time\n",
            "Olympic champion Peres Jepchirchir storms to victory in a women's only world-record time of two hours 16 minutes and 16 seconds at the London Marathon.\n",
            "Great reads\n",
            "Minions and camels hit the streets for marathon\n",
            "More than 50,000 people are running through the capital, many in fancy dress.\n",
            "A new take on matzah ball soup\n",
            "To embrace her Jewish heritage and Mexican upbringing, Fany Gerson adds spicy chillies, avocado, coriander and lime to her matzah ball soup – a perfect twist for a Passover Seder.\n",
            "Must watch\n",
            "Could there soon be digital copies of your heart?\n",
            "Earth's transformation in satellite images\n",
            "Landsat satellites provide the longest continuous global record of the Earth's surface.\n",
            "Why giant hailstones are on the rise\n",
            "Although huge hailstones are still rare, there are signs that their frequency and size are increasing.\n",
            "Listen to world's first 'chat' between humans and whales\n",
            "Scientists had a conversation with Twain, the humpback whale. It's the first ever recorded chat of its kind.\n",
            "The surprising benefits of a traditional Japanese diet\n",
            "New research shows that a traditional Japanese diet rich in fish and soybeans can stave off cognitive decline.\n",
            "Feeling angry? Venting doesn't actually help\n",
            "You might think it's helpful to vent or smash things when you're angry. But a new study shows it doesn't help.\n",
            "Sweden's Icehotel: The menu inspired by eight Sami seasons\n",
            "The menu at the Icehotel, the world's first hotel made out of ice and snow, is served on plates of ice.\n",
            "How AI and deepfakes are changing politics\n",
            "Some politicians have found themselves victims of deepfakes. Can the public trust politicians in the age of AI?\n",
            "How smoking affects your belly fat\n",
            "There's more than just one type of fat - and it's more important than our waistlines.\n",
            "The tiny piece of the US hidden in England\n",
            "How one day in 1963 changed history forever and created a piece of America in the UK.\n",
            "Flying cars: The future of personal mobility\n",
            "Flying cars are no longer a thing of science fiction. Are they the future of daily travel?\n",
            "US English: Why do Brits hate it so?\n",
            "The British sometimes have a few bugbears when it comes to US English spellings, but is it justified?\n",
            "Parthenogenesis: Why some species experience 'virgin births'\n",
            "The rare phenomenon of asexual reproduction continues to perplex scientists.\n",
            "Oatzempic: Should you try the oat drink for weight loss?\n",
            "We asked an expert about oatzempic, the latest viral weight loss trend on TikTok.\n",
            "Space isn't as colourful as photos make it seem\n",
            "The James Webb Telescope sends back stunning photos of our Universe. But are the vibrant colours real?\n",
            "Tracing Marco Polo's footsteps along the Silk Road\n",
            "700 years after his death, Marco Polo's travellogue is full of wonder but also 'hard to believe' in some parts.\n",
            "Slovenia's remote bedroom on top of a snowy mountain\n",
            "Eva Zu Beck heads to the Slovenian mountains where she vists the country's remote refuges.\n",
            "England's 'largest gold nugget' discovered\n",
            "A metal detectorist has uncovered something pretty unexpected and highly valuable on farmland in rural England.\n",
            "Eclipse 2024: What makes it so special?\n",
            "Astrophysicist Erika Hamden explains why this time it's different.\n",
            "Words that reveal if you are an extrovert or an introvert\n",
            "Hyperbole is now a 'mega' part of the English language, but why did it happen?\n",
            "Northern Lights viewing is about to get more magical\n",
            "There are many myths surrounding the aurora but satellite technology provides insights into the celestial marvel.\n",
            "News video\n",
            "'We felt we should honour a mum in our community'\n",
            "Moment US House passes critical Ukraine aid package\n",
            "Moment huge explosion rocks Iraq military base\n",
            "Rio's Christ the Redeemer statue lit up green\n",
            "Cicadas on the menu in New Orleans\n",
            "Business\n",
            "VW workers vote to unionise in major win for labour\n",
            "Dubai airlines resume full schedule after flooding chaos\n",
            "Oil price eases as Iran downplays attack\n",
            "Netflix profits soar after password crackdown\n",
            "US interest rate setter says 'no hurry' to cut\n",
            "Why a deluge of Chinese-made drugs is hard to curb\n",
            "Culture\n",
            "Spice Girls reunite... at Victoria's 50th birthday\n",
            "BBC Proms conductor Sir Andrew Davis dies aged 80\n",
            "12 stunning buildings that bring nature inside\n",
            "Cillian Murphy adds Ifta win to his trophy cabinet\n",
            "Taylor Swift broke Spotify record with new album\n",
            "American Idol singer Mandisa dies aged 47\n",
            "Editor's picks\n",
            "Your pictures on the theme of 'signs of spring'\n",
            "A selection of striking images from our readers around the world.\n",
            "Fallout: Can post-apocalypse show turn Amazon Prime viewers into gamers?\n",
            "Are video game adaptations \"the modern version of seeing the film and then deciding to read the book?\"\n",
            "California's beaches are home to wild fish orgies\n",
            "Millions of small fish fling themselves on Californian beaches to have sex – strictly under the full or new Moon.\n",
            "Earth\n",
            "'I want to erase my own footprint': The women looking after an island paradise\n",
            "'Apocalyptic' Dubai floods shake picture-perfect city\n",
            "Chinese cities sinking under their own weight\n",
            "Rescuers deflate hedgehog with 'balloon' syndrome\n",
            "Did cloud seeding cause the Dubai flooding?\n",
            "Deadly Africa heat caused by human-induced warming\n",
            "Innovation\n",
            "TikTok warns US ban would 'trample free speech'\n",
            "From WhatsApp to Greggs - why is tech going down more?\n",
            "Can AI help solve Japan’s labour shortages?\n",
            "Influential women's tech network shuts down unexpectedly\n",
            "Tesla recalls Cybertrucks over accelerator crash risk\n",
            "The fish that honk to attract potential mates\n",
            "Science and health\n",
            "Does 'zombie deer disease' pose risks for humans?\n",
            "There are growing concerns that chronic wasting disease may be able to jump the species barrier with a possible link to Creutzfeldt-Jakob disease.\n",
            "'People are blamed if they have cancer'\n",
            "Women with south Asian heritage on the adverse reactions they get from other people after diagnosis.\n",
            "Does apple cider vinegar really help with weight loss?\n",
            "Debunking the health benefits of apple cider vinegar.\n",
            "Does it really take 10,000 steps to stay healthy?\n",
            "Studies tout the benefits of walking, yet offer conflicting advice on daily step goals.\n",
            "What is the iron lung and how does it work?\n",
            "The iron lung kept Paul Alexander, known as 'Polio Paul', alive for more than 70 years. But what is it?\n",
            "The 24-year-old selling human bones\n",
            "From a warehouse in Brooklyn, a recent college graduate collects, displays and sells human remains.\n",
            "The search for a cancer 'kill-switch'... in space\n",
            "Cancer cells develop at warp speed in space, potentially aiding some highly experimental new treatments.\n",
            "Why we have hair on our heads\n",
            "Why did humans evolve to retain hair on their heads while losing it on their bodies?\n",
            "Are we worse at Moon landings than 50 years ago?\n",
            "We've set foot on the Moon multiple times. But getting to the lunar surface is far from straightforward.\n",
            "The 17th Century town that quarantined itself\n",
            "The heroic story of a British village that took decisive action when the bubonic plague struck.\n",
            "What to know about brain computers\n",
            "Elon Musk's Neuralink has developed a brain chip that allows users to control a computer with their mind.\n",
            "How to make better New Year's resolutions\n",
            "Have you 'broken' any News Year's resolutions yet? If so, you're not alone - on average 80% of resolutions fail.\n",
            "We inhale a credit card's worth of microplastics each week\n",
            "A new study has found microplastics where they've never been seen before, sitting deep inside human lungs.\n",
            "Does cough medicine actually work?\n",
            "The maker of Robitussin is recalling several products containing honey due to contamination.\n",
            "See what your brain does when you look at art\n",
            "Cutting-edge headsets show the impact of art on human brainwaves.\n",
            "Could this drink really help you sleep?\n",
            "The 'sleepy girl' mocktail is a concoction made of tart cherry juice and magnesium. Here's the science behind it.\n",
            "The hoax equation behind Blue Monday\n",
            "Blue Monday is apparently the most depressing day of the year. Here's the truth about the equation behind it.\n",
            "How I rewired my brain in six weeks\n",
            "There is growing evidence that simple, everyday changes to our lives can alter our brains.\n",
            "How to fight your winter blues as darker days set in\n",
            "Simple tricks to fight your winter blues and restore your energy as darker days set in.\n",
            "How your family shapes your body image\n",
            "The number of people who dislike their body is depressingly high. What can we do about it?\n",
            "An inside look at the near-death experience\n",
            "A new study might explain why dying patients' lives flash before their eyes.\n",
            "What did our ancient Universe sound like?\n",
            "The Planck space telescope has picked up echoes left by soundwaves that travelled through the early Universe.\n",
            "Sport\n",
            "Premier League: Liverpool extend lead at Fulham - radio and text\n",
            "Solo breakaway wins Liege-Bastogne-Liege for Pogacar\n",
            "Man Utd scrape past Coventry to reach FA Cup final\n",
            "'We're back to where we want to be' - Clement\n",
            "London Marathon to Wembley - Ratcliffe's busy Sunday\n",
            "Women's Six Nations: Final round TV coverage & fixtures\n",
            "Travel\n",
            "Where to find the best cheesesteaks in Philly\n",
            "World of Wonder\n",
            "The turtle video that sparked a plastic straw revolution\n",
            "Bloody and in pain, when a plastic straw became lodged in the nostril of a turtle, a video showing its removal shook the world – sparking a movement to rid the world of plastic straws.\n",
            "How a fictitious 'sea' became a top attraction\n",
            "The Maldives' famous Sea of Stars is part fact, part fiction – but that just adds to the mystery of one of the country's top tourist attractions\n",
            "Where to see spring tulips in the Netherlands\n",
            "Nienke Panis-Ringersma has made a career out of following the region's famous blooms. Here are her favourite ways to experience the season, from road trips to bulb picking.\n",
            "Botswana's inspirational women safari guides navigating change\n",
            "A remarkable shift is reshaping the traditional landscape of piloting iconic canoes in one of Africa's most extraordinary places – a role that has long been the domain of men.\n",
            "Photos from the deep sea show 'exciting' new species\n",
            "Scientists exploring the ocean off the New Zealand coast believe they have discovered 100 new marine species.\n",
            "The prehistoric cows making a comeback in Portugal\n",
            "The tauros, a specially bred version of the long-extinct auroch cattle, is being introduced to Portugal's Côa Valley.\n",
            "The US state that's home to Earth's oldest forest\n",
            "The discovery of a 385-million-year-old forest in Cairo, NY, has stunned the world – but visitors to the region have been able to see rare fossil forests for more than a century.\n",
            "Mesmerising life in the deep sea\n",
            "Explore depths where volcanic fluids meet near-freezing seawater, creating a dynamic clash below.\n",
            "Flirt, date, divorce: The robust romantic lives of birds\n",
            "A look at their love lives reveals a unique courtship and the reality of separation.\n",
            "Rare footage shows life through the eyes of polar bears\n",
            "Scientists attached cameras to polar bears to learn how they adapt to a warming planet and found worrying trends.\n",
            "Photographer tells BBC how he captured polar bear photo\n",
            "The poignant image of a snoozing polar bear won the Wildlife Photographer of the Year People's Choice Award.\n",
            "Searching for gold in Alaska\n",
            "Qasa Alom visits Alaska where he searches for gold with a local gold prospector.\n",
            "The monster volcanoes hidden beneath Italy\n",
            "You know about Mount Vesuvius, but there's a nearby 'supervolcano' with even more immense scale and power.\n",
            "Why 1,000 manatees gathered at a hot spring\n",
            "A record number of manatees converged at a Florida state park this week. Experts weigh in on this \"very unusual\" phenomenon.\n",
            "The US is bracing itself for a cicada mega swarm\n",
            "Two broods of cicadas are due to emerge from the ground this April at the same time for the first time in 200 years. It's going to be loud, messy and very interesting.\n",
            "The last male of an extinct species\n",
            "Sudan, the world's last male northern white rhino, became a conservation icon in his final year of life.\n",
            "The last tourism destination on Earth\n",
            "More people are visiting the frozen continent than ever before. Has the very idea of Antarctic tourism become unethical?\n",
            "The plastic oceans we can't see\n",
            "In 1997 Captain Charles Moore was sailing from Hawaii to California when he noticed a steady stream of plastics bobbing in the ocean. He had discovered the Great Pacific Garbage Patch.\n",
            "Why US ranchers are becoming beaver believers\n",
            "For decades, beavers were considered pests – trapped and shot on sight. Now the attitude towards nature's best engineers is changing, and farmers are working to bring them back.\n",
            "The Greek Island of geometrical wonders\n",
            "Pyrgi is a puzzling medieval village in the middle of the Mediterranean island of Chios, in Greece.\n",
            "Sign up for newsletters\n",
            "US Election Unspun\n",
            "Cut through the spin with North America correspondent Anthony Zurcher - in your inbox every Wednesday.\n",
            "In History\n",
            "Discover the headlines from this week in history. See the past come to life through the BBC's unique archive, in your inbox every Thursday.\n",
            "World news\n",
            "Irishman dies while swimming in Portugal\n",
            "Eurovision medal from 1967 sold for £4k at auction\n",
            "Seven killed as car hits crowd at Sri Lanka race\n",
            "Israel investigating death of West Bank paramedic\n",
            "Denmark endures Notre-Dame fire moment and vows to rebuild\n",
            "US and Canada news\n",
            "US Speaker may pay political price for Ukraine deal\n",
            "USC graduation speakers cancelled as tensions flare\n",
            "Man who set fire to himself near Trump trial dies\n",
            "Video of US baseball coach in cockpit under FAA probe\n",
            "After hunt for impartiality, who are the 12 Trump jurors?\n",
            "UK news\n",
            "MP Mark Menzies quits Tories after funds claims\n",
            "'We're the first country to halve emissions'\n",
            "Labour making a 'big commitment' to clean energy\n",
            "Our climate change record is strong, minister says\n",
            "What do GPs think of Sunak's sick note plans?\n",
            "Business\n",
            "Prepayment meter payout numbers unacceptable – minister\n",
            "Vape firm claims it will survive UK disposable ban\n",
            "Look, no hands! My trip on Seoul's self-driving bus\n",
            "VW workers vote to unionise in major win for labour\n",
            "App to cut EU border queues won't be ready in time\n",
            "Culture\n",
            "BBC Proms conductor Sir Andrew Davis dies aged 80\n",
            "Spice Girls reunite... at Victoria's 50th birthday\n",
            "Tram crash at Universal Studios Hollywood injures 15\n",
            "How a 70s make-do-and-mend attitude resonates now\n",
            "UK's largest indoor arena opens with big plans\n",
            "Sport\n",
            "Solo breakaway wins Liege-Bastogne-Liege for Pogacar\n",
            "Man Utd scrape past Coventry to reach FA Cup final\n",
            "London Marathon to Wembley - Ratcliffe's busy Sunday\n",
            "Women's Six Nations: Final round TV coverage & fixtures\n",
            "Liverpool return to form with fine win at Fulham\n",
            "Climate solutions\n",
            "California's beaches are home to wild fish orgies\n",
            "The fish that honk to attract potential mates\n",
            "These are the neighbourhoods in most urgent need of trees\n",
            "This is how you make the world's oldest beer\n",
            "Why rocks on Earth resemble the planet Mercury\n",
            "Technology\n",
            "TikTok warns US ban would 'trample free speech'\n",
            "Fallout: Can post-apocalypse show turn Amazon Prime viewers into gamers?\n",
            "From WhatsApp to Greggs - why is tech going down more?\n",
            "Could there soon be digital copies of your heart?\n",
            "Can AI help solve Japan’s labour shortages?\n",
            "Science and health\n",
            "Dad to run London Marathon after leukaemia diagnosis\n",
            "Ketamine addict: 'I can't walk 50m without weeing'\n",
            "Children used as 'guinea pigs' in UK clinical trials\n",
            "Is filtered water healthier than tap water?\n",
            "Watch: Ancient bones may be thousands of years old\n",
            "World's Table\n",
            "The wines birthed from black volcanic craters\n",
            "Thailand's cooling rice dish to beat the heat\n",
            "The most beautiful cake for Ramadan\n",
            "China's sweet dumpling to remember the dead\n",
            "The 'calorie-bomb' avocado smoothie for Ramadan\n",
            "Travel\n",
            "Where to find the best banh mi in Ho Chi Minh City\n",
            "Eight of the best bagels in New York City\n",
            "A geographer's guide to London's green spaces\n",
            "A chef's guide to the best carbonara in Rome\n",
            "Five under-the-radar eats in Los Angeles\n",
            "Copyright 2024 BBC. All rights reserved.  The BBC is not responsible for the content of external sites. Read about our approach to external linking.\n",
            "\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Print the tokenized text\n",
        "print(\"Tokenized text:\\n\")\n",
        "tokenized_text = tokenizer.tokenize(text)\n",
        "print(tokenized_text)\n",
        "print(\"\\n\")\n",
        "\n",
        "print(\"Summary:\\n\")\n",
        "print(summarize(text))\n",
        "print(\"Word count:\", count_words(summarize(text)), \"\\n\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DLlhrH6Kv1GF",
        "outputId": "46bc0c43-a0e8-4065-f4fd-c8345e5eabf5"
      },
      "id": "DLlhrH6Kv1GF",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Tokenized text:\n",
            "\n",
            "['us', 'house', 'approve', '##s', 'crucial', '$', '61', '##bn', 'war', 'aid', 'for', 'ukraine', 'the', 'vote', 'will', 'be', 'closely', 'watched', 'in', 'kyiv', ',', 'which', 'has', 'warned', 'of', 'an', 'urgent', 'need', 'for', 'more', 'military', 'support', '.', 'the', \"'\", 'robin', 'hood', \"'\", 'ti', '##kt', '##oke', '##r', 'taking', 'on', 'landlord', '##s', 'jordan', 'van', 'den', 'berg', \"'\", 's', 'vi', '##gil', '##ante', '-', 'style', 'approach', 'has', 'won', 'him', 'fans', 'and', 'critics', 'amid', 'a', 'housing', 'crisis', '.', 'us', 'house', 'approve', '##s', 'crucial', '$', '61', '##bn', 'war', 'aid', 'for', 'ukraine', 'the', 'vote', 'will', 'be', 'closely', 'watched', 'in', 'kyiv', ',', 'which', 'has', 'warned', 'of', 'an', 'urgent', 'need', 'for', 'more', 'military', 'support', '.', 'the', \"'\", 'robin', 'hood', \"'\", 'ti', '##kt', '##oke', '##r', 'taking', 'on', 'landlord', '##s', 'jordan', 'van', 'den', 'berg', \"'\", 's', 'vi', '##gil', '##ante', '-', 'style', 'approach', 'has', 'won', 'him', 'fans', 'and', 'critics', 'amid', 'a', 'housing', 'crisis', '.', 'us', 'aid', 'could', 'give', 'ukraine', 'the', 'weapons', 'it', 'needs', 'to', 'slow', 'russia', '’', 's', 'advance', 'the', '$', '61', '##bn', '(', '£', '##49', '##bn', ')', 'aid', 'package', 'keeps', 'ukraine', 'in', 'the', 'fight', 'but', 'it', \"'\", 's', 'no', 'silver', 'bullet', ',', 'bbc', \"'\", 's', 'james', 'water', '##house', 'writes', 'from', 'kyiv', '.', 'what', 'the', 'world', 'makes', 'of', 'trump', 'going', 'on', 'trial', 'pictures', 'of', 'donald', 'trump', 'in', 'a', 'new', 'york', 'courtroom', 'made', 'headlines', 'globally', '.', 'here', \"'\", 's', 'a', 'flavour', 'of', 'what', 'was', 'said', '.', 'us', 'migrant', 'crisis', 'shifts', 'from', 'texas', 'to', 'california', 'border', 'a', 'record', 'surge', 'of', 'illegal', 'border', 'crossings', 'in', 'recent', 'years', 'has', 'made', 'immigration', 'a', 'top', 'voter', 'concern', '.', 'pm', \"'\", 'appalled', \"'\", 'by', 'police', 'treatment', 'of', 'jewish', 'man', 'a', 'no', '10', 'source', 'says', 'police', '\"', 'mis', '##hand', '##led', '\"', 'its', 'interaction', 'with', 'a', 'jewish', 'man', 'at', 'pro', '-', 'palestinian', 'protest', '.', 'je', '##pc', '##hir', '##chi', '##r', 'wins', 'in', 'women', \"'\", 's', 'only', 'world', 'record', 'time', 'olympic', 'champion', 'pere', '##s', 'je', '##pc', '##hir', '##chi', '##r', 'storms', 'to', 'victory', 'in', 'a', 'women', \"'\", 's', 'only', 'world', '-', 'record', 'time', 'of', 'two', 'hours', '16', 'minutes', 'and', '16', 'seconds', 'at', 'the', 'london', 'marathon', '.', 'great', 'reads', 'minions', 'and', 'camel', '##s', 'hit', 'the', 'streets', 'for', 'marathon', 'more', 'than', '50', ',', '000', 'people', 'are', 'running', 'through', 'the', 'capital', ',', 'many', 'in', 'fancy', 'dress', '.', 'a', 'new', 'take', 'on', 'mat', '##za', '##h', 'ball', 'soup', 'to', 'embrace', 'her', 'jewish', 'heritage', 'and', 'mexican', 'upbringing', ',', 'fan', '##y', 'ge', '##rson', 'adds', 'spicy', 'chill', '##ies', ',', 'av', '##oca', '##do', ',', 'co', '##rian', '##der', 'and', 'lime', 'to', 'her', 'mat', '##za', '##h', 'ball', 'soup', '–', 'a', 'perfect', 'twist', 'for', 'a', 'pass', '##over', 'se', '##der', '.', 'must', 'watch', 'could', 'there', 'soon', 'be', 'digital', 'copies', 'of', 'your', 'heart', '?', 'earth', \"'\", 's', 'transformation', 'in', 'satellite', 'images', 'lands', '##at', 'satellites', 'provide', 'the', 'longest', 'continuous', 'global', 'record', 'of', 'the', 'earth', \"'\", 's', 'surface', '.', 'why', 'giant', 'hail', '##stones', 'are', 'on', 'the', 'rise', 'although', 'huge', 'hail', '##stones', 'are', 'still', 'rare', ',', 'there', 'are', 'signs', 'that', 'their', 'frequency', 'and', 'size', 'are', 'increasing', '.', 'listen', 'to', 'world', \"'\", 's', 'first', \"'\", 'chat', \"'\", 'between', 'humans', 'and', 'whales', 'scientists', 'had', 'a', 'conversation', 'with', 'twain', ',', 'the', 'hum', '##p', '##back', 'whale', '.', 'it', \"'\", 's', 'the', 'first', 'ever', 'recorded', 'chat', 'of', 'its', 'kind', '.', 'the', 'surprising', 'benefits', 'of', 'a', 'traditional', 'japanese', 'diet', 'new', 'research', 'shows', 'that', 'a', 'traditional', 'japanese', 'diet', 'rich', 'in', 'fish', 'and', 'soy', '##be', '##ans', 'can', 'st', '##ave', 'off', 'cognitive', 'decline', '.', 'feeling', 'angry', '?', 'vent', '##ing', 'doesn', \"'\", 't', 'actually', 'help', 'you', 'might', 'think', 'it', \"'\", 's', 'helpful', 'to', 'vent', 'or', 'smash', 'things', 'when', 'you', \"'\", 're', 'angry', '.', 'but', 'a', 'new', 'study', 'shows', 'it', 'doesn', \"'\", 't', 'help', '.', 'sweden', \"'\", 's', 'ice', '##hot', '##el', ':', 'the', 'menu', 'inspired', 'by', 'eight', 'sami', 'seasons', 'the', 'menu', 'at', 'the', 'ice', '##hot', '##el', ',', 'the', 'world', \"'\", 's', 'first', 'hotel', 'made', 'out', 'of', 'ice', 'and', 'snow', ',', 'is', 'served', 'on', 'plates', 'of', 'ice', '.', 'how', 'ai', 'and', 'deep', '##fa', '##kes', 'are', 'changing', 'politics', 'some', 'politicians', 'have', 'found', 'themselves', 'victims', 'of', 'deep', '##fa', '##kes', '.', 'can', 'the', 'public', 'trust', 'politicians', 'in', 'the', 'age', 'of', 'ai', '?', 'how', 'smoking', 'affects', 'your', 'belly', 'fat', 'there', \"'\", 's', 'more', 'than', 'just', 'one', 'type', 'of', 'fat', '-', 'and', 'it', \"'\", 's', 'more', 'important', 'than', 'our', 'waist', '##lines', '.', 'the', 'tiny', 'piece', 'of', 'the', 'us', 'hidden', 'in', 'england', 'how', 'one', 'day', 'in', '1963', 'changed', 'history', 'forever', 'and', 'created', 'a', 'piece', 'of', 'america', 'in', 'the', 'uk', '.', 'flying', 'cars', ':', 'the', 'future', 'of', 'personal', 'mobility', 'flying', 'cars', 'are', 'no', 'longer', 'a', 'thing', 'of', 'science', 'fiction', '.', 'are', 'they', 'the', 'future', 'of', 'daily', 'travel', '?', 'us', 'english', ':', 'why', 'do', 'brit', '##s', 'hate', 'it', 'so', '?', 'the', 'british', 'sometimes', 'have', 'a', 'few', 'bug', '##be', '##ars', 'when', 'it', 'comes', 'to', 'us', 'english', 'spelling', '##s', ',', 'but', 'is', 'it', 'justified', '?', 'part', '##hen', '##ogen', '##esis', ':', 'why', 'some', 'species', 'experience', \"'\", 'virgin', 'births', \"'\", 'the', 'rare', 'phenomenon', 'of', 'as', '##ex', '##ual', 'reproduction', 'continues', 'to', 'per', '##plex', 'scientists', '.', 'o', '##atz', '##em', '##pic', ':', 'should', 'you', 'try', 'the', 'o', '##at', 'drink', 'for', 'weight', 'loss', '?', 'we', 'asked', 'an', 'expert', 'about', 'o', '##atz', '##em', '##pic', ',', 'the', 'latest', 'viral', 'weight', 'loss', 'trend', 'on', 'ti', '##kt', '##ok', '.', 'space', 'isn', \"'\", 't', 'as', 'colourful', 'as', 'photos', 'make', 'it', 'seem', 'the', 'james', 'webb', 'telescope', 'sends', 'back', 'stunning', 'photos', 'of', 'our', 'universe', '.', 'but', 'are', 'the', 'vibrant', 'colours', 'real', '?', 'tracing', 'marco', 'polo', \"'\", 's', 'footsteps', 'along', 'the', 'silk', 'road', '700', 'years', 'after', 'his', 'death', ',', 'marco', 'polo', \"'\", 's', 'travel', '##logue', 'is', 'full', 'of', 'wonder', 'but', 'also', \"'\", 'hard', 'to', 'believe', \"'\", 'in', 'some', 'parts', '.', 'slovenia', \"'\", 's', 'remote', 'bedroom', 'on', 'top', 'of', 'a', 'snowy', 'mountain', 'eva', 'zu', 'beck', 'heads', 'to', 'the', 'slovenian', 'mountains', 'where', 'she', 'vis', '##ts', 'the', 'country', \"'\", 's', 'remote', 'refuge', '##s', '.', 'england', \"'\", 's', \"'\", 'largest', 'gold', 'nu', '##gg', '##et', \"'\", 'discovered', 'a', 'metal', 'detector', '##ist', 'has', 'uncovered', 'something', 'pretty', 'unexpected', 'and', 'highly', 'valuable', 'on', 'farmland', 'in', 'rural', 'england', '.', 'eclipse', '202', '##4', ':', 'what', 'makes', 'it', 'so', 'special', '?', 'astro', '##phy', '##sic', '##ist', 'erika', 'ham', '##den', 'explains', 'why', 'this', 'time', 'it', \"'\", 's', 'different', '.', 'words', 'that', 'reveal', 'if', 'you', 'are', 'an', 'ex', '##tro', '##vert', 'or', 'an', 'intro', '##vert', 'hyper', '##bol', '##e', 'is', 'now', 'a', \"'\", 'mega', \"'\", 'part', 'of', 'the', 'english', 'language', ',', 'but', 'why', 'did', 'it', 'happen', '?', 'northern', 'lights', 'viewing', 'is', 'about', 'to', 'get', 'more', 'magical', 'there', 'are', 'many', 'myths', 'surrounding', 'the', 'aurora', 'but', 'satellite', 'technology', 'provides', 'insights', 'into', 'the', 'celestial', 'marvel', '.', 'news', 'video', \"'\", 'we', 'felt', 'we', 'should', 'honour', 'a', 'mum', 'in', 'our', 'community', \"'\", 'moment', 'us', 'house', 'passes', 'critical', 'ukraine', 'aid', 'package', 'moment', 'huge', 'explosion', 'rocks', 'iraq', 'military', 'base', 'rio', \"'\", 's', 'christ', 'the', 'red', '##eem', '##er', 'statue', 'lit', 'up', 'green', 'ci', '##ca', '##das', 'on', 'the', 'menu', 'in', 'new', 'orleans', 'business', 'v', '##w', 'workers', 'vote', 'to', 'union', '##ise', 'in', 'major', 'win', 'for', 'labour', 'dubai', 'airlines', 'resume', 'full', 'schedule', 'after', 'flooding', 'chaos', 'oil', 'price', 'ease', '##s', 'as', 'iran', 'down', '##play', '##s', 'attack', 'netflix', 'profits', 'so', '##ar', 'after', 'password', 'crack', '##down', 'us', 'interest', 'rate', 'set', '##ter', 'says', \"'\", 'no', 'hurry', \"'\", 'to', 'cut', 'why', 'a', 'del', '##uge', 'of', 'chinese', '-', 'made', 'drugs', 'is', 'hard', 'to', 'curb', 'culture', 'spice', 'girls', 'reunite', '.', '.', '.', 'at', 'victoria', \"'\", 's', '50th', 'birthday', 'bbc', 'prom', '##s', 'conductor', 'sir', 'andrew', 'davis', 'dies', 'aged', '80', '12', 'stunning', 'buildings', 'that', 'bring', 'nature', 'inside', 'ci', '##lli', '##an', 'murphy', 'adds', 'if', '##ta', 'win', 'to', 'his', 'trophy', 'cabinet', 'taylor', 'swift', 'broke', 'spot', '##ify', 'record', 'with', 'new', 'album', 'american', 'idol', 'singer', 'man', '##dis', '##a', 'dies', 'aged', '47', 'editor', \"'\", 's', 'picks', 'your', 'pictures', 'on', 'the', 'theme', 'of', \"'\", 'signs', 'of', 'spring', \"'\", 'a', 'selection', 'of', 'striking', 'images', 'from', 'our', 'readers', 'around', 'the', 'world', '.', 'fallout', ':', 'can', 'post', '-', 'apocalypse', 'show', 'turn', 'amazon', 'prime', 'viewers', 'into', 'gamer', '##s', '?', 'are', 'video', 'game', 'adaptations', '\"', 'the', 'modern', 'version', 'of', 'seeing', 'the', 'film', 'and', 'then', 'deciding', 'to', 'read', 'the', 'book', '?', '\"', 'california', \"'\", 's', 'beaches', 'are', 'home', 'to', 'wild', 'fish', 'org', '##ies', 'millions', 'of', 'small', 'fish', 'fling', 'themselves', 'on', 'california', '##n', 'beaches', 'to', 'have', 'sex', '–', 'strictly', 'under', 'the', 'full', 'or', 'new', 'moon', '.', 'earth', \"'\", 'i', 'want', 'to', 'erase', 'my', 'own', 'footprint', \"'\", ':', 'the', 'women', 'looking', 'after', 'an', 'island', 'paradise', \"'\", 'apocalyptic', \"'\", 'dubai', 'floods', 'shake', 'picture', '-', 'perfect', 'city', 'chinese', 'cities', 'sinking', 'under', 'their', 'own', 'weight', 'rescue', '##rs', 'def', '##late', 'hedge', '##hog', 'with', \"'\", 'balloon', \"'\", 'syndrome', 'did', 'cloud', 'seed', '##ing', 'cause', 'the', 'dubai', 'flooding', '?', 'deadly', 'africa', 'heat', 'caused', 'by', 'human', '-', 'induced', 'warming', 'innovation', 'ti', '##kt', '##ok', 'warns', 'us', 'ban', 'would', \"'\", 'tram', '##ple', 'free', 'speech', \"'\", 'from', 'what', '##sa', '##pp', 'to', 'gregg', '##s', '-', 'why', 'is', 'tech', 'going', 'down', 'more', '?', 'can', 'ai', 'help', 'solve', 'japan', '’', 's', 'labour', 'shortages', '?', 'influential', 'women', \"'\", 's', 'tech', 'network', 'shut', '##s', 'down', 'unexpectedly', 'tesla', 'recalls', 'cyber', '##tr', '##uck', '##s', 'over', 'accelerator', 'crash', 'risk', 'the', 'fish', 'that', 'hon', '##k', 'to', 'attract', 'potential', 'mates', 'science', 'and', 'health', 'does', \"'\", 'zombie', 'deer', 'disease', \"'\", 'pose', 'risks', 'for', 'humans', '?', 'there', 'are', 'growing', 'concerns', 'that', 'chronic', 'wasting', 'disease', 'may', 'be', 'able', 'to', 'jump', 'the', 'species', 'barrier', 'with', 'a', 'possible', 'link', 'to', 'cr', '##eu', '##tz', '##feld', '##t', '-', 'jakob', 'disease', '.', \"'\", 'people', 'are', 'blamed', 'if', 'they', 'have', 'cancer', \"'\", 'women', 'with', 'south', 'asian', 'heritage', 'on', 'the', 'adverse', 'reactions', 'they', 'get', 'from', 'other', 'people', 'after', 'diagnosis', '.', 'does', 'apple', 'cid', '##er', 'vinegar', 'really', 'help', 'with', 'weight', 'loss', '?', 'de', '##bu', '##nk', '##ing', 'the', 'health', 'benefits', 'of', 'apple', 'cid', '##er', 'vinegar', '.', 'does', 'it', 'really', 'take', '10', ',', '000', 'steps', 'to', 'stay', 'healthy', '?', 'studies', 'to', '##ut', 'the', 'benefits', 'of', 'walking', ',', 'yet', 'offer', 'conflicting', 'advice', 'on', 'daily', 'step', 'goals', '.', 'what', 'is', 'the', 'iron', 'lung', 'and', 'how', 'does', 'it', 'work', '?', 'the', 'iron', 'lung', 'kept', 'paul', 'alexander', ',', 'known', 'as', \"'\", 'pol', '##io', 'paul', \"'\", ',', 'alive', 'for', 'more', 'than', '70', 'years', '.', 'but', 'what', 'is', 'it', '?', 'the', '24', '-', 'year', '-', 'old', 'selling', 'human', 'bones', 'from', 'a', 'warehouse', 'in', 'brooklyn', ',', 'a', 'recent', 'college', 'graduate', 'collects', ',', 'displays', 'and', 'sells', 'human', 'remains', '.', 'the', 'search', 'for', 'a', 'cancer', \"'\", 'kill', '-', 'switch', \"'\", '.', '.', '.', 'in', 'space', 'cancer', 'cells', 'develop', 'at', 'warp', 'speed', 'in', 'space', ',', 'potentially', 'aiding', 'some', 'highly', 'experimental', 'new', 'treatments', '.', 'why', 'we', 'have', 'hair', 'on', 'our', 'heads', 'why', 'did', 'humans', 'evolve', 'to', 'retain', 'hair', 'on', 'their', 'heads', 'while', 'losing', 'it', 'on', 'their', 'bodies', '?', 'are', 'we', 'worse', 'at', 'moon', 'landings', 'than', '50', 'years', 'ago', '?', 'we', \"'\", 've', 'set', 'foot', 'on', 'the', 'moon', 'multiple', 'times', '.', 'but', 'getting', 'to', 'the', 'lunar', 'surface', 'is', 'far', 'from', 'straightforward', '.', 'the', '17th', 'century', 'town', 'that', 'qu', '##aran', '##tine', '##d', 'itself', 'the', 'heroic', 'story', 'of', 'a', 'british', 'village', 'that', 'took', 'decisive', 'action', 'when', 'the', 'bu', '##bon', '##ic', 'plague', 'struck', '.', 'what', 'to', 'know', 'about', 'brain', 'computers', 'el', '##on', 'mu', '##sk', \"'\", 's', 'neural', '##ink', 'has', 'developed', 'a', 'brain', 'chip', 'that', 'allows', 'users', 'to', 'control', 'a', 'computer', 'with', 'their', 'mind', '.', 'how', 'to', 'make', 'better', 'new', 'year', \"'\", 's', 'resolutions', 'have', 'you', \"'\", 'broken', \"'\", 'any', 'news', 'year', \"'\", 's', 'resolutions', 'yet', '?', 'if', 'so', ',', 'you', \"'\", 're', 'not', 'alone', '-', 'on', 'average', '80', '%', 'of', 'resolutions', 'fail', '.', 'we', 'in', '##hale', 'a', 'credit', 'card', \"'\", 's', 'worth', 'of', 'micro', '##pl', '##astic', '##s', 'each', 'week', 'a', 'new', 'study', 'has', 'found', 'micro', '##pl', '##astic', '##s', 'where', 'they', \"'\", 've', 'never', 'been', 'seen', 'before', ',', 'sitting', 'deep', 'inside', 'human', 'lungs', '.', 'does', 'cough', 'medicine', 'actually', 'work', '?', 'the', 'maker', 'of', 'rob', '##it', '##uss', '##in', 'is', 'recalling', 'several', 'products', 'containing', 'honey', 'due', 'to', 'contamination', '.', 'see', 'what', 'your', 'brain', 'does', 'when', 'you', 'look', 'at', 'art', 'cutting', '-', 'edge', 'heads', '##ets', 'show', 'the', 'impact', 'of', 'art', 'on', 'human', 'brain', '##wave', '##s', '.', 'could', 'this', 'drink', 'really', 'help', 'you', 'sleep', '?', 'the', \"'\", 'sleepy', 'girl', \"'\", 'mock', '##tail', 'is', 'a', 'con', '##co', '##ction', 'made', 'of', 'tar', '##t', 'cherry', 'juice', 'and', 'magnesium', '.', 'here', \"'\", 's', 'the', 'science', 'behind', 'it', '.', 'the', 'hoax', 'equation', 'behind', 'blue', 'monday', 'blue', 'monday', 'is', 'apparently', 'the', 'most', 'de', '##pressing', 'day', 'of', 'the', 'year', '.', 'here', \"'\", 's', 'the', 'truth', 'about', 'the', 'equation', 'behind', 'it', '.', 'how', 'i', 're', '##wire', '##d', 'my', 'brain', 'in', 'six', 'weeks', 'there', 'is', 'growing', 'evidence', 'that', 'simple', ',', 'everyday', 'changes', 'to', 'our', 'lives', 'can', 'alter', 'our', 'brains', '.', 'how', 'to', 'fight', 'your', 'winter', 'blues', 'as', 'darker', 'days', 'set', 'in', 'simple', 'tricks', 'to', 'fight', 'your', 'winter', 'blues', 'and', 'restore', 'your', 'energy', 'as', 'darker', 'days', 'set', 'in', '.', 'how', 'your', 'family', 'shapes', 'your', 'body', 'image', 'the', 'number', 'of', 'people', 'who', 'dislike', 'their', 'body', 'is', 'de', '##pressing', '##ly', 'high', '.', 'what', 'can', 'we', 'do', 'about', 'it', '?', 'an', 'inside', 'look', 'at', 'the', 'near', '-', 'death', 'experience', 'a', 'new', 'study', 'might', 'explain', 'why', 'dying', 'patients', \"'\", 'lives', 'flash', 'before', 'their', 'eyes', '.', 'what', 'did', 'our', 'ancient', 'universe', 'sound', 'like', '?', 'the', 'planck', 'space', 'telescope', 'has', 'picked', 'up', 'echoes', 'left', 'by', 'sound', '##wave', '##s', 'that', 'travelled', 'through', 'the', 'early', 'universe', '.', 'sport', 'premier', 'league', ':', 'liverpool', 'extend', 'lead', 'at', 'fulham', '-', 'radio', 'and', 'text', 'solo', 'breakaway', 'wins', 'liege', '-', 'bas', '##to', '##gne', '-', 'liege', 'for', 'po', '##ga', '##car', 'man', 'ut', '##d', 'scrape', 'past', 'coventry', 'to', 'reach', 'fa', 'cup', 'final', \"'\", 'we', \"'\", 're', 'back', 'to', 'where', 'we', 'want', 'to', 'be', \"'\", '-', 'clement', 'london', 'marathon', 'to', 'wembley', '-', 'rat', '##cliffe', \"'\", 's', 'busy', 'sunday', 'women', \"'\", 's', 'six', 'nations', ':', 'final', 'round', 'tv', 'coverage', '&', 'fixtures', 'travel', 'where', 'to', 'find', 'the', 'best', 'cheese', '##ste', '##aks', 'in', 'phil', '##ly', 'world', 'of', 'wonder', 'the', 'turtle', 'video', 'that', 'sparked', 'a', 'plastic', 'straw', 'revolution', 'bloody', 'and', 'in', 'pain', ',', 'when', 'a', 'plastic', 'straw', 'became', 'lodged', 'in', 'the', 'nos', '##tri', '##l', 'of', 'a', 'turtle', ',', 'a', 'video', 'showing', 'its', 'removal', 'shook', 'the', 'world', '–', 'spark', '##ing', 'a', 'movement', 'to', 'rid', 'the', 'world', 'of', 'plastic', 'straw', '##s', '.', 'how', 'a', 'fictitious', \"'\", 'sea', \"'\", 'became', 'a', 'top', 'attraction', 'the', 'maldives', \"'\", 'famous', 'sea', 'of', 'stars', 'is', 'part', 'fact', ',', 'part', 'fiction', '–', 'but', 'that', 'just', 'adds', 'to', 'the', 'mystery', 'of', 'one', 'of', 'the', 'country', \"'\", 's', 'top', 'tourist', 'attractions', 'where', 'to', 'see', 'spring', 'tu', '##lip', '##s', 'in', 'the', 'netherlands', 'ni', '##en', '##ke', 'pan', '##is', '-', 'ring', '##ers', '##ma', 'has', 'made', 'a', 'career', 'out', 'of', 'following', 'the', 'region', \"'\", 's', 'famous', 'blooms', '.', 'here', 'are', 'her', 'favourite', 'ways', 'to', 'experience', 'the', 'season', ',', 'from', 'road', 'trips', 'to', 'bulb', 'picking', '.', 'botswana', \"'\", 's', 'inspirational', 'women', 'safari', 'guides', 'na', '##vi', '##gating', 'change', 'a', 'remarkable', 'shift', 'is', 'res', '##ha', '##ping', 'the', 'traditional', 'landscape', 'of', 'pilot', '##ing', 'iconic', 'canoe', '##s', 'in', 'one', 'of', 'africa', \"'\", 's', 'most', 'extraordinary', 'places', '–', 'a', 'role', 'that', 'has', 'long', 'been', 'the', 'domain', 'of', 'men', '.', 'photos', 'from', 'the', 'deep', 'sea', 'show', \"'\", 'exciting', \"'\", 'new', 'species', 'scientists', 'exploring', 'the', 'ocean', 'off', 'the', 'new', 'zealand', 'coast', 'believe', 'they', 'have', 'discovered', '100', 'new', 'marine', 'species', '.', 'the', 'prehistoric', 'cows', 'making', 'a', 'comeback', 'in', 'portugal', 'the', 'tau', '##ros', ',', 'a', 'specially', 'bred', 'version', 'of', 'the', 'long', '-', 'extinct', 'au', '##ro', '##ch', 'cattle', ',', 'is', 'being', 'introduced', 'to', 'portugal', \"'\", 's', 'coa', 'valley', '.', 'the', 'us', 'state', 'that', \"'\", 's', 'home', 'to', 'earth', \"'\", 's', 'oldest', 'forest', 'the', 'discovery', 'of', 'a', '385', '-', 'million', '-', 'year', '-', 'old', 'forest', 'in', 'cairo', ',', 'ny', ',', 'has', 'stunned', 'the', 'world', '–', 'but', 'visitors', 'to', 'the', 'region', 'have', 'been', 'able', 'to', 'see', 'rare', 'fossil', 'forests', 'for', 'more', 'than', 'a', 'century', '.', 'me', '##sm', '##eri', '##sing', 'life', 'in', 'the', 'deep', 'sea', 'explore', 'depths', 'where', 'volcanic', 'fluids', 'meet', 'near', '-', 'freezing', 'sea', '##water', ',', 'creating', 'a', 'dynamic', 'clash', 'below', '.', 'flirt', ',', 'date', ',', 'divorce', ':', 'the', 'robust', 'romantic', 'lives', 'of', 'birds', 'a', 'look', 'at', 'their', 'love', 'lives', 'reveals', 'a', 'unique', 'courtship', 'and', 'the', 'reality', 'of', 'separation', '.', 'rare', 'footage', 'shows', 'life', 'through', 'the', 'eyes', 'of', 'polar', 'bears', 'scientists', 'attached', 'cameras', 'to', 'polar', 'bears', 'to', 'learn', 'how', 'they', 'adapt', 'to', 'a', 'warming', 'planet', 'and', 'found', 'worrying', 'trends', '.', 'photographer', 'tells', 'bbc', 'how', 'he', 'captured', 'polar', 'bear', 'photo', 'the', 'po', '##ignant', 'image', 'of', 'a', 's', '##no', '##oz', '##ing', 'polar', 'bear', 'won', 'the', 'wildlife', 'photographer', 'of', 'the', 'year', 'people', \"'\", 's', 'choice', 'award', '.', 'searching', 'for', 'gold', 'in', 'alaska', 'q', '##asa', 'al', '##om', 'visits', 'alaska', 'where', 'he', 'searches', 'for', 'gold', 'with', 'a', 'local', 'gold', 'prospect', '##or', '.', 'the', 'monster', 'volcanoes', 'hidden', 'beneath', 'italy', 'you', 'know', 'about', 'mount', 've', '##su', '##vius', ',', 'but', 'there', \"'\", 's', 'a', 'nearby', \"'\", 'super', '##vo', '##lc', '##ano', \"'\", 'with', 'even', 'more', 'immense', 'scale', 'and', 'power', '.', 'why', '1', ',', '000', 'mana', '##tees', 'gathered', 'at', 'a', 'hot', 'spring', 'a', 'record', 'number', 'of', 'mana', '##tees', 'converge', '##d', 'at', 'a', 'florida', 'state', 'park', 'this', 'week', '.', 'experts', 'weigh', 'in', 'on', 'this', '\"', 'very', 'unusual', '\"', 'phenomenon', '.', 'the', 'us', 'is', 'bracing', 'itself', 'for', 'a', 'ci', '##ca', '##da', 'mega', 'swarm', 'two', 'brood', '##s', 'of', 'ci', '##ca', '##das', 'are', 'due', 'to', 'emerge', 'from', 'the', 'ground', 'this', 'april', 'at', 'the', 'same', 'time', 'for', 'the', 'first', 'time', 'in', '200', 'years', '.', 'it', \"'\", 's', 'going', 'to', 'be', 'loud', ',', 'messy', 'and', 'very', 'interesting', '.', 'the', 'last', 'male', 'of', 'an', 'extinct', 'species', 'sudan', ',', 'the', 'world', \"'\", 's', 'last', 'male', 'northern', 'white', 'rhino', ',', 'became', 'a', 'conservation', 'icon', 'in', 'his', 'final', 'year', 'of', 'life', '.', 'the', 'last', 'tourism', 'destination', 'on', 'earth', 'more', 'people', 'are', 'visiting', 'the', 'frozen', 'continent', 'than', 'ever', 'before', '.', 'has', 'the', 'very', 'idea', 'of', 'antarctic', 'tourism', 'become', 'une', '##thic', '##al', '?', 'the', 'plastic', 'oceans', 'we', 'can', \"'\", 't', 'see', 'in', '1997', 'captain', 'charles', 'moore', 'was', 'sailing', 'from', 'hawaii', 'to', 'california', 'when', 'he', 'noticed', 'a', 'steady', 'stream', 'of', 'plastics', 'bob', '##bing', 'in', 'the', 'ocean', '.', 'he', 'had', 'discovered', 'the', 'great', 'pacific', 'garbage', 'patch', '.', 'why', 'us', 'ranch', '##ers', 'are', 'becoming', 'beaver', 'believers', 'for', 'decades', ',', 'beaver', '##s', 'were', 'considered', 'pest', '##s', '–', 'trapped', 'and', 'shot', 'on', 'sight', '.', 'now', 'the', 'attitude', 'towards', 'nature', \"'\", 's', 'best', 'engineers', 'is', 'changing', ',', 'and', 'farmers', 'are', 'working', 'to', 'bring', 'them', 'back', '.', 'the', 'greek', 'island', 'of', 'geometric', '##al', 'wonders', 'p', '##yr', '##gi', 'is', 'a', 'pu', '##zzling', 'medieval', 'village', 'in', 'the', 'middle', 'of', 'the', 'mediterranean', 'island', 'of', 'chi', '##os', ',', 'in', 'greece', '.', 'sign', 'up', 'for', 'newsletter', '##s', 'us', 'election', 'un', '##sp', '##un', 'cut', 'through', 'the', 'spin', 'with', 'north', 'america', 'correspondent', 'anthony', 'zur', '##cher', '-', 'in', 'your', 'in', '##box', 'every', 'wednesday', '.', 'in', 'history', 'discover', 'the', 'headlines', 'from', 'this', 'week', 'in', 'history', '.', 'see', 'the', 'past', 'come', 'to', 'life', 'through', 'the', 'bbc', \"'\", 's', 'unique', 'archive', ',', 'in', 'your', 'in', '##box', 'every', 'thursday', '.', 'world', 'news', 'irish', '##man', 'dies', 'while', 'swimming', 'in', 'portugal', 'eurovision', 'medal', 'from', '1967', 'sold', 'for', '£', '##4', '##k', 'at', 'auction', 'seven', 'killed', 'as', 'car', 'hits', 'crowd', 'at', 'sri', 'lanka', 'race', 'israel', 'investigating', 'death', 'of', 'west', 'bank', 'para', '##med', '##ic', 'denmark', 'endure', '##s', 'notre', '-', 'dame', 'fire', 'moment', 'and', 'vows', 'to', 'rebuild', 'us', 'and', 'canada', 'news', 'us', 'speaker', 'may', 'pay', 'political', 'price', 'for', 'ukraine', 'deal', 'usc', 'graduation', 'speakers', 'cancelled', 'as', 'tensions', 'flare', 'man', 'who', 'set', 'fire', 'to', 'himself', 'near', 'trump', 'trial', 'dies', 'video', 'of', 'us', 'baseball', 'coach', 'in', 'cockpit', 'under', 'faa', 'probe', 'after', 'hunt', 'for', 'imp', '##art', '##ial', '##ity', ',', 'who', 'are', 'the', '12', 'trump', 'ju', '##ror', '##s', '?', 'uk', 'news', 'mp', 'mark', 'men', '##zie', '##s', 'quit', '##s', 'tori', '##es', 'after', 'funds', 'claims', \"'\", 'we', \"'\", 're', 'the', 'first', 'country', 'to', 'hal', '##ve', 'emissions', \"'\", 'labour', 'making', 'a', \"'\", 'big', 'commitment', \"'\", 'to', 'clean', 'energy', 'our', 'climate', 'change', 'record', 'is', 'strong', ',', 'minister', 'says', 'what', 'do', 'gps', 'think', 'of', 'sun', '##ak', \"'\", 's', 'sick', 'note', 'plans', '?', 'business', 'prep', '##ay', '##ment', 'meter', 'pay', '##out', 'numbers', 'unacceptable', '–', 'minister', 'va', '##pe', 'firm', 'claims', 'it', 'will', 'survive', 'uk', 'di', '##sp', '##osa', '##ble', 'ban', 'look', ',', 'no', 'hands', '!', 'my', 'trip', 'on', 'seoul', \"'\", 's', 'self', '-', 'driving', 'bus', 'v', '##w', 'workers', 'vote', 'to', 'union', '##ise', 'in', 'major', 'win', 'for', 'labour', 'app', 'to', 'cut', 'eu', 'border', 'queue', '##s', 'won', \"'\", 't', 'be', 'ready', 'in', 'time', 'culture', 'bbc', 'prom', '##s', 'conductor', 'sir', 'andrew', 'davis', 'dies', 'aged', '80', 'spice', 'girls', 'reunite', '.', '.', '.', 'at', 'victoria', \"'\", 's', '50th', 'birthday', 'tram', 'crash', 'at', 'universal', 'studios', 'hollywood', 'in', '##jure', '##s', '15', 'how', 'a', '70s', 'make', '-', 'do', '-', 'and', '-', 'men', '##d', 'attitude', 'res', '##onate', '##s', 'now', 'uk', \"'\", 's', 'largest', 'indoor', 'arena', 'opens', 'with', 'big', 'plans', 'sport', 'solo', 'breakaway', 'wins', 'liege', '-', 'bas', '##to', '##gne', '-', 'liege', 'for', 'po', '##ga', '##car', 'man', 'ut', '##d', 'scrape', 'past', 'coventry', 'to', 'reach', 'fa', 'cup', 'final', 'london', 'marathon', 'to', 'wembley', '-', 'rat', '##cliffe', \"'\", 's', 'busy', 'sunday', 'women', \"'\", 's', 'six', 'nations', ':', 'final', 'round', 'tv', 'coverage', '&', 'fixtures', 'liverpool', 'return', 'to', 'form', 'with', 'fine', 'win', 'at', 'fulham', 'climate', 'solutions', 'california', \"'\", 's', 'beaches', 'are', 'home', 'to', 'wild', 'fish', 'org', '##ies', 'the', 'fish', 'that', 'hon', '##k', 'to', 'attract', 'potential', 'mates', 'these', 'are', 'the', 'neighbourhoods', 'in', 'most', 'urgent', 'need', 'of', 'trees', 'this', 'is', 'how', 'you', 'make', 'the', 'world', \"'\", 's', 'oldest', 'beer', 'why', 'rocks', 'on', 'earth', 'resemble', 'the', 'planet', 'mercury', 'technology', 'ti', '##kt', '##ok', 'warns', 'us', 'ban', 'would', \"'\", 'tram', '##ple', 'free', 'speech', \"'\", 'fallout', ':', 'can', 'post', '-', 'apocalypse', 'show', 'turn', 'amazon', 'prime', 'viewers', 'into', 'gamer', '##s', '?', 'from', 'what', '##sa', '##pp', 'to', 'gregg', '##s', '-', 'why', 'is', 'tech', 'going', 'down', 'more', '?', 'could', 'there', 'soon', 'be', 'digital', 'copies', 'of', 'your', 'heart', '?', 'can', 'ai', 'help', 'solve', 'japan', '’', 's', 'labour', 'shortages', '?', 'science', 'and', 'health', 'dad', 'to', 'run', 'london', 'marathon', 'after', 'le', '##uka', '##emia', 'diagnosis', 'ke', '##tam', '##ine', 'addict', ':', \"'\", 'i', 'can', \"'\", 't', 'walk', '50', '##m', 'without', 'wee', '##ing', \"'\", 'children', 'used', 'as', \"'\", 'guinea', 'pigs', \"'\", 'in', 'uk', 'clinical', 'trials', 'is', 'filtered', 'water', 'health', '##ier', 'than', 'tap', 'water', '?', 'watch', ':', 'ancient', 'bones', 'may', 'be', 'thousands', 'of', 'years', 'old', 'world', \"'\", 's', 'table', 'the', 'wines', 'birth', '##ed', 'from', 'black', 'volcanic', 'craters', 'thailand', \"'\", 's', 'cooling', 'rice', 'dish', 'to', 'beat', 'the', 'heat', 'the', 'most', 'beautiful', 'cake', 'for', 'rama', '##dan', 'china', \"'\", 's', 'sweet', 'dump', '##ling', 'to', 'remember', 'the', 'dead', 'the', \"'\", 'cal', '##ori', '##e', '-', 'bomb', \"'\", 'av', '##oca', '##do', 'smooth', '##ie', 'for', 'rama', '##dan', 'travel', 'where', 'to', 'find', 'the', 'best', 'ban', '##h', 'mi', 'in', 'ho', 'chi', 'minh', 'city', 'eight', 'of', 'the', 'best', 'bag', '##els', 'in', 'new', 'york', 'city', 'a', 'geo', '##grapher', \"'\", 's', 'guide', 'to', 'london', \"'\", 's', 'green', 'spaces', 'a', 'chef', \"'\", 's', 'guide', 'to', 'the', 'best', 'carbon', '##ara', 'in', 'rome', 'five', 'under', '-', 'the', '-', 'radar', 'eats', 'in', 'los', 'angeles', 'copyright', '202', '##4', 'bbc', '.', 'all', 'rights', 'reserved', '.', 'the', 'bbc', 'is', 'not', 'responsible', 'for', 'the', 'content', 'of', 'external', 'sites', '.', 'read', 'about', 'our', 'approach', 'to', 'external', 'linking', '.']\n",
            "\n",
            "\n",
            "Summary:\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/sklearn/cluster/_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "US House approves crucial $61bn war aid for Ukraine\n",
            "The vote will be closely watched in Kyiv, which has warned of an urgent need for more military support \n",
            "How to make better New Year's resolutions\n",
            "Have you 'broken' any News Year's resolutions yet? If so, you're not alone - on average 80% of resolutions fail  Here's the truth about the equation behind it \n",
            "Sport\n",
            "Premier League: Liverpool extend lead at Fulham - radio and text\n",
            "Solo breakaway wins Liege-Bastogne-Liege for Pogacar\n",
            "Man Utd scrape past Coventry to reach FA Cup final\n",
            "'We're back to where we want to be' - Clement\n",
            "London Marathon to Wembley - Ratcliffe's busy Sunday\n",
            "Women's Six Nations: Final round TV coverage & fixtures\n",
            "Travel\n",
            "Where to find the best cheesesteaks in Philly\n",
            "World of Wonder\n",
            "The turtle video that sparked a plastic straw revolution\n",
            "Bloody and in pain, when a plastic straw became lodged in the nostril of a turtle, a video showing its removal shook the world – sparking a movement to rid the world of plastic straws  But are the vibrant colours real?\n",
            "Tracing Marco Polo's footsteps along the Silk Road\n",
            "700 years after his death, Marco Polo's travellogue is full of wonder but also 'hard to believe' in some parts  Can the public trust politicians in the age of AI?\n",
            "How smoking affects your belly fat\n",
            "There's more than just one type of fat - and it's more important than our waistlines \n",
            "Space isn't as colourful as photos make it seem\n",
            "The James Webb Telescope sends back stunning photos of our Universe  Has the very idea of Antarctic tourism become unethical?\n",
            "The plastic oceans we can't see\n",
            "In 1997 Captain Charles Moore was sailing from Hawaii to California when he noticed a steady stream of plastics bobbing in the ocean  \n",
            "Sign up for newsletters\n",
            "US Election Unspun\n",
            "Cut through the spin with North America correspondent Anthony Zurcher - in your inbox every Wednesday\n",
            "Word count: 318 \n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/sklearn/cluster/_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#sequence classification"
      ],
      "metadata": {
        "id": "Y_Urk8SRw253"
      },
      "id": "Y_Urk8SRw253"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6e0c0c5e",
      "metadata": {
        "id": "6e0c0c5e",
        "outputId": "f265d4b4-b7b0-4d3c-ea52-bdf04eb58ada",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:429: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "name 'dataloader' is not defined",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-22-9578325f3a06>\u001b[0m in \u001b[0;36m<cell line: 17>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0mnum_epochs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_epochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0mbatch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdataloader\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m         \u001b[0minput_ids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'input_ids'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'dataloader' is not defined"
          ]
        }
      ],
      "source": [
        "from transformers import BertForSequenceClassification, BertTokenizer, AdamW\n",
        "import torch\n",
        "\n",
        "# Load pre-trained BERT model and tokenizer\n",
        "model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "\n",
        "# Prepare your dataset and dataloaders for fine-tuning\n",
        "# (Note: You need to provide your own dataset and dataloaders)\n",
        "\n",
        "# Define optimizer and learning rate\n",
        "optimizer = AdamW(model.parameters(), lr=5e-5)\n",
        "\n",
        "# Fine-tune the BERT model\n",
        "model.train()\n",
        "num_epochs = 3\n",
        "for epoch in range(num_epochs):\n",
        "    for batch in dataloader:\n",
        "        optimizer.zero_grad()\n",
        "        input_ids = batch['input_ids']\n",
        "        attention_mask = batch['attention_mask']\n",
        "        labels = batch['labels']\n",
        "        outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n",
        "        loss = outputs.loss\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "# Evaluate the fine-tuned model\n",
        "model.eval()\n",
        "# Run evaluation on a separate test set\n",
        "\n",
        "# Save the fine-tuned model for future use\n",
        "model.save_pretrained('fine_tuned_bert_model')"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.7"
    },
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 5
}