train_sklearn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd \n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = pd.read_csv(\"data/train.csv\")\n",
    "test_data = pd.read_csv(\"data/test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "split_word = [x.split(' ') for x in list(train_data['text'])]\n",
    "split_word_test = [x.split(' ') for x in list(test_data['text'])]\n",
    "train_data['text_split'] = split_word\n",
    "test_data['text_split'] = split_word_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "train, valid = train_test_split(train_data, test_size = 0.2, random_state = 42) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.to_csv('data/train_use.csv', index=False)\n",
    "valid.to_csv('data/valid_use.csv', index=False)\n",
    "test_data.to_csv('data/test_use.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sklearn "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"data/train_use.csv\")\n",
    "valid = pd.read_csv(\"data/valid_use.csv\")\n",
    "test = pd.read_csv(\"data/test_use.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import HashingVectorizer\n",
    "hv = HashingVectorizer(n_features=5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6090\n",
      "1523\n",
      "3263\n",
      "10876\n"
     ]
    }
   ],
   "source": [
    "X_raw = [str(train['keyword'][i]) + ' ' + train['text'][i] for i in range(len(train['keyword']))]\n",
    "X_raw_valid = [str(valid['keyword'][i]) + ' ' + valid['text'][i] for i in range(len(valid['keyword']))]\n",
    "X_raw_test = [str(test['keyword'][i]) + ' ' + test['text'][i] for i in range(len(test['keyword']))]\n",
    "print(len(X_raw))\n",
    "print(len(X_raw_valid))\n",
    "print(len(X_raw_test))\n",
    "print(len(X_raw) + len(X_raw_valid) + len(X_raw_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nt = pd.DataFrame()\\nt['words'] = X_raw\\nt['target'] = list(train['target'])\\nv = pd.DataFrame()\\nv['words'] = X_raw_valid\\nv['target'] = list(valid['target'])\\nte = pd.DataFrame()\\nte['words'] = X_raw_test\\nt.to_csv('data/train_rnn.csv', index=False)\\nv.to_csv('data/valid_rnn.csv', index=False)\\nte.to_csv('data/test_rnn.csv', index=False)\\n\""
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "t = pd.DataFrame()\n",
    "t['words'] = X_raw\n",
    "t['target'] = list(train['target'])\n",
    "v = pd.DataFrame()\n",
    "v['words'] = X_raw_valid\n",
    "v['target'] = list(valid['target'])\n",
    "te = pd.DataFrame()\n",
    "te['words'] = X_raw_test\n",
    "t.to_csv('data/train_rnn.csv', index=False)\n",
    "v.to_csv('data/valid_rnn.csv', index=False)\n",
    "te.to_csv('data/test_rnn.csv', index=False)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10876\n"
     ]
    }
   ],
   "source": [
    "X_whole = X_raw.copy()\n",
    "X_whole.extend(X_raw_valid)\n",
    "X_whole.extend(X_raw_test)\n",
    "print(len(X_whole))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10876, 5000)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = hv.transform(X_whole).toarray()\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X[:6090]\n",
    "X_valid = X[6090:(6090+1523)]\n",
    "X_test = X[(6090+1523):]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = list(train['target'])\n",
    "y_valid = list(valid['target'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "X_train_pd = pd.DataFrame(X_train)\n",
    "X_train_pd['target'] = y_train\n",
    "X_train_pd.to_csv('data/train_lr.csv', index=False)\n",
    "X_valid_pd = pd.DataFrame(X_valid)\n",
    "X_valid_pd['target'] = y_valid\n",
    "X_valid_pd.to_csv('data/valid_lr.csv', index=False)\n",
    "X_test_pd = pd.DataFrame(X_test)\n",
    "X_test_pd.to_csv('data/test_lr.csv', index=False)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7839789888378201"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr = LogisticRegression(random_state=0).fit(X_train, y_train)\n",
    "lr.score(X_valid, y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3263"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_predict = lr.predict(X_test)\n",
    "len(lr_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_df = pd.DataFrame({\"id\": test_data[\"id\"], \"target\": lr_predict})\n",
    "preds_df.to_csv(f\"data/output_lr_15000.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "f2d0b0f03ac3a581d6b1b551bd30a19f866dd2d205f22e8dba00531575c12bcc"
  },
  "kernelspec": {
   "display_name": "Python 3.8.12 64-bit ('.venv': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}