Newsela_Paragraph_Alignment_Dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/phillab/anaconda2/envs/py36/lib/python3.6/site-packages/ipykernel_launcher.py:13: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
      "  del sys.path[0]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "38d6256238e947e2b2098445e0227896",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "import csv, os, tqdm, nltk\n",
    "import sqlite3, random\n",
    "\n",
    "# This folder can be obtained directly from the Newsela team\n",
    "# by contacting them at: https://newsela.com/about/resources/research/\n",
    "\n",
    "newsela_folder = \"/home/phillab/dataset/newsela_article_corpus_2016-01-29/\"\n",
    "\n",
    "keys = ['slug', 'language', 'title', 'grade_level', 'version', 'filename']\n",
    "slug2articles = {}\n",
    "min_passage_length = 40\n",
    "\n",
    "with open(os.path.join(newsela_folder, \"articles_metadata.csv\"), \"r\") as f:\n",
    "    objs = csv.reader(f)\n",
    "    for i, obj in tqdm.tqdm_notebook(enumerate(objs)):\n",
    "        if i == 0: continue\n",
    "\n",
    "        article = {k: obj[j] for j, k in enumerate(keys)}\n",
    "        if article['language'] != \"en\":\n",
    "            continue\n",
    "\n",
    "        article['grade_level'] = float(article['grade_level'])\n",
    "        article['version'] = float(article['version'])\n",
    "        \n",
    "        with open(os.path.join(newsela_folder, \"articles/\"+article['filename']), \"r\") as f2:\n",
    "            article['content'] = f2.read()\n",
    "        paragraphs = [p.strip() for p in article['content'].split(\"\\n\\n\") if \"##\" not in p]\n",
    "        \n",
    "        article['paragraphs'] = []\n",
    "        build_up = \"\"\n",
    "        for p in paragraphs:\n",
    "            build_up += p+\"\\n\\n\"\n",
    "            if build_up.count(\" \") > min_passage_length:\n",
    "                article['paragraphs'].append(build_up.strip())\n",
    "                build_up = \"\"\n",
    "                \n",
    "        # The last bit\n",
    "        if build_up.count(\" \") > min_passage_length:\n",
    "            article['paragraphs'].append(build_up.strip())\n",
    "        \n",
    "        if article['slug'] not in slug2articles:\n",
    "            slug2articles[article['slug']] = []\n",
    "        slug2articles[article['slug']].append(article)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "version_paired = \"0.2\"\n",
    "dataset_file = \"/home/phillab/dataset/newsela/newsela_paired_%s.db\" % (version_paired)\n",
    "if os.path.isfile(dataset_file):\n",
    "    os.remove(dataset_file)\n",
    "conn = sqlite3.connect(dataset_file, detect_types=sqlite3.PARSE_DECLTYPES)\n",
    "conn.row_factory = sqlite3.Row\n",
    "c = conn.cursor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CREATE TABLE\n",
    "\n",
    "sql_create = \"CREATE TABLE articles (id INTEGER PRIMARY KEY AUTOINCREMENT, p1 TEXT NOT NULL, p2 TEXT NOT NULL, version1 INTEGER NOT NULL,  version2 INTEGER NOT NULL, slug TEXT NOT NULL, cut TEXT NOT NULL);\"\n",
    "c.execute(sql_create)\n",
    "\n",
    "c.execute(\"CREATE INDEX idx_source ON articles(cut);\")\n",
    "c.execute(\"CREATE INDEX idx_date ON articles(slug);\")\n",
    "\n",
    "conn.commit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/phillab/anaconda2/envs/py36/lib/python3.6/site-packages/ipykernel_launcher.py:5: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
      "  \"\"\"\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bf28418ff0604137bcf3209ef14e569d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1911.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Number of same samples: 10125\n"
     ]
    }
   ],
   "source": [
    "import Levenshtein, utils_misc\n",
    "sql_insert = \"INSERT INTO articles (p1, p2, version1, version2, slug, cut) VALUES (?, ?, ?, ?, ?, ?)\"\n",
    "\n",
    "same = 0\n",
    "for i, slug in enumerate(tqdm.tqdm_notebook(slug2articles)):\n",
    "    cut = \"dev\" if i%40 == 0 else \"train\"\n",
    "    arts = slug2articles[slug]\n",
    "    for a in arts:\n",
    "        a['paras'] = [p.strip() for p in a['content'].split(\"\\n\\n\") if \"##\" not in p] # Own non-merged paragraphs\n",
    "        a['paras'] = [p for p in a['paras'] if p.count(\" \") >= 18 and p.count(\" \") <= 80]\n",
    "\n",
    "    for a1 in arts:\n",
    "        for a2 in arts:\n",
    "            if a1['version'] >= a2['version']:\n",
    "                continue\n",
    "            for p1 in a1['paras']:\n",
    "                for p2 in a2['paras']:\n",
    "                    R = float(len(p1)) / len(p2)\n",
    "                    if R <= 0.6 or R >= 1.5 or p1 == p2:\n",
    "                        continue\n",
    "\n",
    "                    ratio = Levenshtein.ratio(p1, p2)\n",
    "                    if ratio >= 0.6 and ratio <= 0.92:\n",
    "#                         print(\"[%.3f] %s\" % (ratio, utils_misc.show_diff(p1, p2)))\n",
    "                        c.execute(sql_insert, (p1, p2, a1['version'], a2['version'], slug, cut))\n",
    "                    if ratio >= 0.97: # assume it is the same\n",
    "                        c.execute(sql_insert, (p1, p2, a1['version'], a1['version'], slug, cut))\n",
    "                        same += 1\n",
    "\n",
    "print(\"Number of same samples: %d\" % (same))\n",
    "conn.commit()"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}