2.3. DataManipulation_Tokenization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Data Manipulation\n",
    "## 2.1. Tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run \"2.0. DataManipulation_Functions.ipynb\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading the spaCy trained pipeline for Russian https://spacy.io/models/ru\n",
    "nlp = spacy.load(\"ru_core_news_sm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "folder = 'data'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Useful functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenization_pipe(df):\n",
    "    \"\"\"\n",
    "    Function the return a list of lists of tokens. It ignores\n",
    "    punctation and stop words\n",
    "\n",
    "    Args: a cleaned dataframe e.g. df_scraped\n",
    "    Returns: a list of list of lists of tokens. Each nested list correspond to a row of the dataframe df\n",
    "    \"\"\"\n",
    "    lemma_text_list = []\n",
    "    for doc in nlp.pipe(df[\"body\"]):\n",
    "        lemma_text_list.append([token.lemma_ for token in doc if (token.is_stop==False and token.is_punct==False)])\n",
    "    return lemma_text_list\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenization_pipe_title(df):\n",
    "    \"\"\"\n",
    "    Function the return a list of lists of tokens. It ignores\n",
    "    punctation and stop words\n",
    "\n",
    "    Args: a cleaned dataframe e.g. df_scraped\n",
    "    Returns: a list of list of lists of tokens. Each nested list correspond to a row of the dataframe df\n",
    "    \"\"\"\n",
    "    lemma_text_list = []\n",
    "    for doc in nlp.pipe(df[\"title\"]):\n",
    "        lemma_text_list.append([token.lemma_ for token in doc if (token.is_stop==False and token.is_punct==False)])\n",
    "    return lemma_text_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_df_scraped_tokenized(df):\n",
    "    \"\"\" add two columns to the df_scraped for title and body news with token. Before that, it creates a list and \n",
    "    then assign this list to the df.\n",
    "    args\n",
    "        df:: df_scraped, the scraped dataframe\n",
    "    return\n",
    "        df:: df_scraped with tokens in body and title. Create two additional columns.\n",
    "    \n",
    "    ***WARNING***\n",
    "    Running this function takes lot of time e.g. 8 hours\n",
    "    \"\"\"\n",
    "    # Running time: 485 minutes and 38.5 sec\n",
    "    list_token_body = tokenization_pipe(df) \n",
    "    df['body_token_final'] = list_token_body\n",
    "    # Running time: ?\n",
    "    list_token_title = tokenization_pipe_title(df)\n",
    "    df['title_token_final'] = list_token_title\n",
    "    \n",
    "    return df \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def df_scraped_tokenized_to_csv(folder, df):\n",
    "    \"\"\"Exporting the df with token into a csv\n",
    "    args\n",
    "        folder:: the pc location\n",
    "        df:: df scraped\n",
    "    \"\"\"\n",
    "    df = get_df_scraped_tokenized(df)\n",
    "    # df.to_csv(os.path.join(folder, 'df_scraped_token.csv'))\n",
    "    df.to_csv(os.path.join(folder, 'df_tokenized.csv'))    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get scraped data and clean it\n",
    "df = load_clean_scraped_data(folder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tokenized = get_df_scraped_tokenized(df)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "vscode": {
   "interpreter": {
    "hash": "ad2bdc8ecc057115af97d19610ffacc2b4e99fae6737bb82f5d7fb13d2f2c186"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}