Preprocessing_data_and_Training_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "EXTRACTING THE AUDIO FILES FROM .TAR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np \n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extraction completed! Files are saved in data\n"
     ]
    }
   ],
   "source": [
    "import tarfile\n",
    "\n",
    "# Define the path to the .tar.gz file\n",
    "tar_file_path = \"speech_commands_v0.02.tar.gz\"\n",
    "\n",
    "# Define the extraction directory\n",
    "extraction_dir = \"data\"\n",
    "\n",
    "# Create the extraction directory if it doesn't exist\n",
    "os.makedirs(extraction_dir, exist_ok=True)\n",
    "\n",
    "# Open the tar.gz file\n",
    "with tarfile.open(tar_file_path, \"r:gz\") as tar:\n",
    "    # Extract all contents\n",
    "    tar.extractall(path=extraction_dir)\n",
    "\n",
    "print(f\"Extraction completed! Files are saved in {extraction_dir}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MAKING A DATAFRAME OF ALL THE AUDIO FILES, WITH THEIR CATEGORY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique categories: 36\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "def create_dataframe_from_folder(main_folder_path):\n",
    "    # Initialize lists to hold file paths and categories\n",
    "    file_paths = []\n",
    "    categories = []\n",
    "\n",
    "    # Iterate through subfolders in the main folder\n",
    "    for subfolder in os.listdir(main_folder_path):\n",
    "        subfolder_path = os.path.join(main_folder_path, subfolder)\n",
    "        \n",
    "        # Check if it's a directory\n",
    "        if os.path.isdir(subfolder_path):\n",
    "            # Check if there are any .wav files in the subfolder\n",
    "            wav_files = [f for f in os.listdir(subfolder_path) if f.endswith('.wav')]\n",
    "            \n",
    "            if wav_files:  # If there are .wav files\n",
    "                # Add each .wav file's path and category to lists\n",
    "                for wav_file in wav_files:\n",
    "                    file_paths.append(os.path.join(subfolder_path, wav_file))\n",
    "                    categories.append(subfolder)\n",
    "    \n",
    "    # Create a DataFrame\n",
    "    df = pd.DataFrame({\n",
    "        'file_path': file_paths,\n",
    "        'category': categories\n",
    "    })\n",
    "\n",
    "    # Count unique categories\n",
    "    unique_categories = df['category'].nunique()\n",
    "    \n",
    "    # Print the number of unique categories\n",
    "    print(f\"Number of unique categories: {unique_categories}\")\n",
    "    \n",
    "    return df\n",
    "\n",
    "# Example usage\n",
    "main_folder_path = 'data'\n",
    "df = create_dataframe_from_folder(main_folder_path)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(105835, 2)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing chunk 1\n",
      "Processing chunk 2\n",
      "Processing chunk 3\n",
      "Processing chunk 4\n",
      "Processing chunk 5\n",
      "Processing chunk 6\n",
      "Processing chunk 7\n",
      "Processing chunk 8\n",
      "Processing chunk 9\n",
      "Processing chunk 10\n",
      "Processing chunk 11\n",
      "Processing chunk 12\n",
      "Processing chunk 13\n",
      "Processing chunk 14\n",
      "Processing chunk 15\n",
      "Processing chunk 16\n",
      "Processing chunk 17\n",
      "Processing chunk 18\n",
      "Processing chunk 19\n",
      "Processing chunk 20\n",
      "Processing chunk 21\n",
      "Processing chunk 22\n",
      "Processing chunk 23\n",
      "Processing chunk 24\n",
      "Processing chunk 25\n",
      "Processing chunk 26\n",
      "Processing chunk 27\n",
      "Processing chunk 28\n",
      "Processing chunk 29\n",
      "Processing chunk 30\n",
      "Processing chunk 31\n",
      "Processing chunk 32\n",
      "Processing chunk 33\n",
      "Processing chunk 34\n",
      "Processing chunk 35\n",
      "Processing chunk 36\n",
      "Processing chunk 37\n",
      "Processing chunk 38\n",
      "Processing chunk 39\n",
      "Processing chunk 40\n",
      "Processing chunk 41\n",
      "Processing chunk 42\n",
      "Processing chunk 43\n",
      "Processing chunk 44\n",
      "Processing chunk 45\n",
      "Processing chunk 46\n",
      "Processing chunk 47\n",
      "Processing chunk 48\n",
      "Processing chunk 49\n",
      "Processing chunk 50\n",
      "Processing chunk 51\n",
      "Processing chunk 52\n",
      "Processing chunk 53\n",
      "Processing chunk 54\n",
      "Processing chunk 55\n",
      "Processing chunk 56\n",
      "Processing chunk 57\n",
      "Processing chunk 58\n",
      "Processing chunk 59\n",
      "Processing chunk 60\n",
      "Processing chunk 61\n",
      "Processing chunk 62\n",
      "Processing chunk 63\n",
      "Processing chunk 64\n",
      "Processing chunk 65\n",
      "Processing chunk 66\n",
      "Processing chunk 67\n",
      "Processing chunk 68\n",
      "Processing chunk 69\n",
      "Processing chunk 70\n",
      "Processing chunk 71\n",
      "Processing chunk 72\n",
      "Processing chunk 73\n",
      "Processing chunk 74\n",
      "Processing chunk 75\n",
      "Processing chunk 76\n",
      "Processing chunk 77\n",
      "Processing chunk 78\n",
      "Processing chunk 79\n",
      "Processing chunk 80\n",
      "Processing chunk 81\n",
      "Processing chunk 82\n",
      "Processing chunk 83\n",
      "Processing chunk 84\n",
      "Processing chunk 85\n",
      "Processing chunk 86\n",
      "Processing chunk 87\n",
      "Processing chunk 88\n",
      "Processing chunk 89\n",
      "Processing chunk 90\n",
      "Processing chunk 91\n",
      "Processing chunk 92\n",
      "Processing chunk 93\n",
      "Processing chunk 94\n",
      "Processing chunk 95\n",
      "Processing chunk 96\n",
      "Processing chunk 97\n",
      "Processing chunk 98\n",
      "Processing chunk 99\n",
      "Processing chunk 100\n",
      "Processing chunk 101\n",
      "Processing chunk 102\n",
      "Processing chunk 103\n",
      "Processing chunk 104\n",
      "Processing chunk 105\n",
      "Processing chunk 106\n",
      "Processing chunk 107\n",
      "Processing chunk 108\n",
      "Processing chunk 109\n",
      "Processing chunk 110\n",
      "Processing chunk 111\n",
      "Processing chunk 112\n",
      "Processing chunk 113\n",
      "Processing chunk 114\n",
      "Processing chunk 115\n",
      "Processing chunk 116\n",
      "Processing chunk 117\n",
      "Processing chunk 118\n",
      "Processing chunk 119\n",
      "Processing chunk 120\n",
      "Processing chunk 121\n",
      "Processing chunk 122\n",
      "Processing chunk 123\n",
      "Processing chunk 124\n",
      "Processing chunk 125\n",
      "Processing chunk 126\n",
      "Processing chunk 127\n",
      "Processing chunk 128\n",
      "Processing chunk 129\n",
      "Processing chunk 130\n",
      "Processing chunk 131\n",
      "Processing chunk 132\n",
      "Processing chunk 133\n",
      "Processing chunk 134\n",
      "Processing chunk 135\n",
      "Processing chunk 136\n",
      "Processing chunk 137\n",
      "Processing chunk 138\n",
      "Processing chunk 139\n",
      "Processing chunk 140\n",
      "Processing chunk 141\n",
      "Processing chunk 142\n",
      "Processing chunk 143\n",
      "Processing chunk 144\n",
      "Processing chunk 145\n",
      "Processing chunk 146\n",
      "Processing chunk 147\n",
      "Processing chunk 148\n",
      "Processing chunk 149\n",
      "Processing chunk 150\n",
      "Processing chunk 151\n",
      "Processing chunk 152\n",
      "Processing chunk 153\n",
      "Processing chunk 154\n",
      "Processing chunk 155\n",
      "Processing chunk 156\n",
      "Processing chunk 157\n",
      "Processing chunk 158\n",
      "Processing chunk 159\n",
      "Processing chunk 160\n",
      "Processing chunk 161\n",
      "Processing chunk 162\n",
      "Processing chunk 163\n",
      "Processing chunk 164\n",
      "Processing chunk 165\n",
      "Processing chunk 166\n",
      "Processing chunk 167\n",
      "Processing chunk 168\n",
      "Processing chunk 169\n",
      "Processing chunk 170\n",
      "Processing chunk 171\n",
      "Processing chunk 172\n",
      "Processing chunk 173\n",
      "Processing chunk 174\n",
      "Processing chunk 175\n",
      "Processing chunk 176\n",
      "Processing chunk 177\n",
      "Processing chunk 178\n",
      "Processing chunk 179\n",
      "Processing chunk 180\n",
      "Processing chunk 181\n",
      "Processing chunk 182\n",
      "Processing chunk 183\n",
      "Processing chunk 184\n",
      "Processing chunk 185\n",
      "Processing chunk 186\n",
      "Processing chunk 187\n",
      "Processing chunk 188\n",
      "Processing chunk 189\n",
      "Processing chunk 190\n",
      "Processing chunk 191\n",
      "Processing chunk 192\n",
      "Processing chunk 193\n",
      "Processing chunk 194\n",
      "Processing chunk 195\n",
      "Processing chunk 196\n",
      "Processing chunk 197\n",
      "Processing chunk 198\n",
      "Processing chunk 199\n",
      "Processing chunk 200\n",
      "Processing chunk 201\n",
      "Processing chunk 202\n",
      "Processing chunk 203\n",
      "Processing chunk 204\n",
      "Processing chunk 205\n",
      "Processing chunk 206\n",
      "Processing chunk 207\n",
      "Processing chunk 208\n",
      "Processing chunk 209\n",
      "Processing chunk 210\n",
      "Processing chunk 211\n",
      "Processing chunk 212\n",
      "Processing chunk 213\n",
      "Processing chunk 214\n",
      "Processing chunk 215\n",
      "Processing chunk 216\n",
      "Processing chunk 217\n",
      "Processing chunk 218\n",
      "Processing chunk 219\n",
      "Processing chunk 220\n",
      "Processing chunk 221\n",
      "Processing chunk 222\n",
      "Processing chunk 223\n",
      "Processing chunk 224\n",
      "Processing chunk 225\n",
      "Processing chunk 226\n",
      "Processing chunk 227\n",
      "Processing chunk 228\n",
      "Processing chunk 229\n",
      "Processing chunk 230\n",
      "Processing chunk 231\n",
      "Processing chunk 232\n",
      "Processing chunk 233\n",
      "Processing chunk 234\n",
      "Processing chunk 235\n",
      "Processing chunk 236\n",
      "Processing chunk 237\n",
      "Processing chunk 238\n",
      "Processing chunk 239\n",
      "Processing chunk 240\n",
      "Processing chunk 241\n",
      "Processing chunk 242\n",
      "Processing chunk 243\n",
      "Processing chunk 244\n",
      "Processing chunk 245\n",
      "Processing chunk 246\n",
      "Processing chunk 247\n",
      "Processing chunk 248\n",
      "Processing chunk 249\n",
      "Processing chunk 250\n",
      "Processing chunk 251\n",
      "Processing chunk 252\n",
      "Processing chunk 253\n",
      "Processing chunk 254\n",
      "Processing chunk 255\n",
      "Processing chunk 256\n",
      "Processing chunk 257\n",
      "Processing chunk 258\n",
      "Processing chunk 259\n",
      "Processing chunk 260\n",
      "Processing chunk 261\n",
      "Processing chunk 262\n",
      "Processing chunk 263\n",
      "Processing chunk 264\n",
      "Processing chunk 265\n",
      "Processing chunk 266\n",
      "Processing chunk 267\n",
      "Processing chunk 268\n",
      "Processing chunk 269\n",
      "Processing chunk 270\n",
      "Processing chunk 271\n",
      "Processing chunk 272\n",
      "Processing chunk 273\n",
      "Processing chunk 274\n",
      "Processing chunk 275\n",
      "Processing chunk 276\n",
      "Processing chunk 277\n",
      "Processing chunk 278\n",
      "Processing chunk 279\n",
      "Processing chunk 280\n",
      "Processing chunk 281\n",
      "Processing chunk 282\n",
      "Processing chunk 283\n",
      "Processing chunk 284\n",
      "Processing chunk 285\n",
      "Processing chunk 286\n",
      "Processing chunk 287\n",
      "Processing chunk 288\n",
      "Processing chunk 289\n",
      "Processing chunk 290\n",
      "Processing chunk 291\n",
      "Processing chunk 292\n",
      "Processing chunk 293\n",
      "Processing chunk 294\n",
      "Processing chunk 295\n",
      "Processing chunk 296\n",
      "Processing chunk 297\n",
      "Processing chunk 298\n",
      "Processing chunk 299\n",
      "Processing chunk 300\n",
      "Processing chunk 301\n",
      "Processing chunk 302\n",
      "Processing chunk 303\n",
      "Processing chunk 304\n",
      "Processing chunk 305\n",
      "Processing chunk 306\n",
      "Processing chunk 307\n",
      "Processing chunk 308\n",
      "Processing chunk 309\n",
      "Processing chunk 310\n",
      "Processing chunk 311\n",
      "Processing chunk 312\n",
      "Processing chunk 313\n",
      "Processing chunk 314\n",
      "Processing chunk 315\n",
      "Processing chunk 316\n",
      "Processing chunk 317\n",
      "Processing chunk 318\n",
      "Processing chunk 319\n",
      "Processing chunk 320\n",
      "Processing chunk 321\n",
      "Processing chunk 322\n",
      "Processing chunk 323\n",
      "Processing chunk 324\n",
      "Processing chunk 325\n",
      "Processing chunk 326\n",
      "Processing chunk 327\n",
      "Processing chunk 328\n",
      "Processing chunk 329\n",
      "Processing chunk 330\n",
      "Processing chunk 331\n",
      "Processing chunk 332\n",
      "Processing chunk 333\n",
      "Processing chunk 334\n",
      "Processing chunk 335\n",
      "Processing chunk 336\n",
      "Processing chunk 337\n",
      "Processing chunk 338\n",
      "Processing chunk 339\n",
      "Processing chunk 340\n",
      "Processing chunk 341\n",
      "Processing chunk 342\n",
      "Processing chunk 343\n",
      "Processing chunk 344\n",
      "Processing chunk 345\n",
      "Processing chunk 346\n",
      "Processing chunk 347\n",
      "Processing chunk 348\n",
      "Processing chunk 349\n",
      "Processing chunk 350\n",
      "Processing chunk 351\n",
      "Processing chunk 352\n",
      "Processing chunk 353\n",
      "Processing chunk 354\n",
      "Processing chunk 355\n",
      "Processing chunk 356\n",
      "Processing chunk 357\n",
      "Processing chunk 358\n",
      "Processing chunk 359\n",
      "Processing chunk 360\n",
      "Processing chunk 361\n",
      "Processing chunk 362\n",
      "Processing chunk 363\n",
      "Processing chunk 364\n",
      "Processing chunk 365\n",
      "Processing chunk 366\n",
      "Processing chunk 367\n",
      "Processing chunk 368\n",
      "Processing chunk 369\n",
      "Processing chunk 370\n",
      "Processing chunk 371\n",
      "Processing chunk 372\n",
      "Processing chunk 373\n",
      "Processing chunk 374\n",
      "Processing chunk 375\n",
      "Processing chunk 376\n",
      "Processing chunk 377\n",
      "Processing chunk 378\n",
      "Processing chunk 379\n",
      "Processing chunk 380\n",
      "Processing chunk 381\n",
      "Processing chunk 382\n",
      "Processing chunk 383\n",
      "Processing chunk 384\n",
      "Processing chunk 385\n",
      "Processing chunk 386\n",
      "Processing chunk 387\n",
      "Processing chunk 388\n",
      "Processing chunk 389\n",
      "Processing chunk 390\n",
      "Processing chunk 391\n",
      "Processing chunk 392\n",
      "Processing chunk 393\n",
      "Processing chunk 394\n",
      "Processing chunk 395\n",
      "Processing chunk 396\n",
      "Processing chunk 397\n",
      "Processing chunk 398\n",
      "Processing chunk 399\n",
      "Processing chunk 400\n",
      "Processing chunk 401\n",
      "Processing chunk 402\n",
      "Processing chunk 403\n",
      "Processing chunk 404\n",
      "Processing chunk 405\n",
      "Processing chunk 406\n",
      "Processing chunk 407\n",
      "Processing chunk 408\n",
      "Processing chunk 409\n",
      "Processing chunk 410\n",
      "Processing chunk 411\n",
      "Processing chunk 412\n",
      "Processing chunk 413\n",
      "Processing chunk 414\n",
      "Processing chunk 415\n",
      "Processing chunk 416\n",
      "Processing chunk 417\n",
      "Processing chunk 418\n",
      "Processing chunk 419\n",
      "Processing chunk 420\n",
      "Processing chunk 421\n",
      "Processing chunk 422\n",
      "Processing chunk 423\n",
      "Processing chunk 424\n",
      "Processing chunk 425\n",
      "Processing chunk 426\n",
      "Processing chunk 427\n",
      "Processing chunk 428\n",
      "Processing chunk 429\n",
      "Processing chunk 430\n",
      "Processing chunk 431\n",
      "Processing chunk 432\n",
      "Processing chunk 433\n",
      "Processing chunk 434\n",
      "Processing chunk 435\n",
      "Processing chunk 436\n",
      "Processing chunk 437\n",
      "Processing chunk 438\n",
      "Processing chunk 439\n",
      "Processing chunk 440\n",
      "Processing chunk 441\n",
      "Processing chunk 442\n",
      "Processing chunk 443\n",
      "Processing chunk 444\n",
      "Processing chunk 445\n",
      "Processing chunk 446\n",
      "Processing chunk 447\n",
      "Processing chunk 448\n",
      "Processing chunk 449\n",
      "Processing chunk 450\n",
      "Processing chunk 451\n",
      "Processing chunk 452\n",
      "Processing chunk 453\n",
      "Processing chunk 454\n",
      "Processing chunk 455\n",
      "Processing chunk 456\n",
      "Processing chunk 457\n",
      "Processing chunk 458\n",
      "Processing chunk 459\n",
      "Processing chunk 460\n",
      "Processing chunk 461\n",
      "Processing chunk 462\n",
      "Processing chunk 463\n",
      "Processing chunk 464\n",
      "Processing chunk 465\n",
      "Processing chunk 466\n",
      "Processing chunk 467\n",
      "Processing chunk 468\n",
      "Processing chunk 469\n",
      "Processing chunk 470\n",
      "Processing chunk 471\n",
      "Processing chunk 472\n",
      "Processing chunk 473\n",
      "Processing chunk 474\n",
      "Processing chunk 475\n",
      "Processing chunk 476\n",
      "Processing chunk 477\n",
      "Processing chunk 478\n",
      "Processing chunk 479\n",
      "Processing chunk 480\n",
      "Processing chunk 481\n",
      "Processing chunk 482\n",
      "Processing chunk 483\n",
      "Processing chunk 484\n",
      "Processing chunk 485\n",
      "Processing chunk 486\n",
      "Processing chunk 487\n",
      "Processing chunk 488\n",
      "Processing chunk 489\n",
      "Processing chunk 490\n",
      "Processing chunk 491\n",
      "Processing chunk 492\n",
      "Processing chunk 493\n",
      "Processing chunk 494\n",
      "Processing chunk 495\n",
      "Processing chunk 496\n",
      "Processing chunk 497\n",
      "Processing chunk 498\n",
      "Processing chunk 499\n",
      "Processing chunk 500\n",
      "Processing chunk 501\n",
      "Processing chunk 502\n",
      "Processing chunk 503\n",
      "Processing chunk 504\n",
      "Processing chunk 505\n",
      "Processing chunk 506\n",
      "Processing chunk 507\n",
      "Processing chunk 508\n",
      "Processing chunk 509\n",
      "Processing chunk 510\n",
      "Processing chunk 511\n",
      "Processing chunk 512\n",
      "Processing chunk 513\n",
      "Processing chunk 514\n",
      "Processing chunk 515\n",
      "Processing chunk 516\n",
      "Processing chunk 517\n",
      "Processing chunk 518\n",
      "Processing chunk 519\n",
      "Processing chunk 520\n",
      "Processing chunk 521\n",
      "Processing chunk 522\n",
      "Processing chunk 523\n",
      "Processing chunk 524\n",
      "Processing chunk 525\n",
      "Processing chunk 526\n",
      "Processing chunk 527\n",
      "Processing chunk 528\n",
      "Processing chunk 529\n",
      "Processing chunk 530\n",
      "Processing chunk 531\n",
      "Processing chunk 532\n",
      "Processing chunk 533\n",
      "Processing chunk 534\n",
      "Processing chunk 535\n",
      "Processing chunk 536\n",
      "Processing chunk 537\n",
      "Processing chunk 538\n",
      "Processing chunk 539\n",
      "Processing chunk 540\n",
      "Processing chunk 541\n",
      "Processing chunk 542\n",
      "Processing chunk 543\n",
      "Processing chunk 544\n",
      "Processing chunk 545\n",
      "Processing chunk 546\n",
      "Processing chunk 547\n",
      "Processing chunk 548\n",
      "Processing chunk 549\n",
      "Processing chunk 550\n",
      "Processing chunk 551\n",
      "Processing chunk 552\n",
      "Processing chunk 553\n",
      "Processing chunk 554\n",
      "Processing chunk 555\n",
      "Processing chunk 556\n",
      "Processing chunk 557\n",
      "Processing chunk 558\n",
      "Processing chunk 559\n",
      "Processing chunk 560\n",
      "Processing chunk 561\n",
      "Processing chunk 562\n",
      "Processing chunk 563\n",
      "Processing chunk 564\n",
      "Processing chunk 565\n",
      "Processing chunk 566\n",
      "Processing chunk 567\n",
      "Processing chunk 568\n",
      "Processing chunk 569\n",
      "Processing chunk 570\n",
      "Processing chunk 571\n",
      "Processing chunk 572\n",
      "Processing chunk 573\n",
      "Processing chunk 574\n",
      "Processing chunk 575\n",
      "Processing chunk 576\n",
      "Processing chunk 577\n",
      "Processing chunk 578\n",
      "Processing chunk 579\n",
      "Processing chunk 580\n",
      "Processing chunk 581\n",
      "Processing chunk 582\n",
      "Processing chunk 583\n",
      "Processing chunk 584\n",
      "Processing chunk 585\n",
      "Processing chunk 586\n",
      "Processing chunk 587\n",
      "Processing chunk 588\n",
      "Processing chunk 589\n",
      "Processing chunk 590\n",
      "Processing chunk 591\n",
      "Processing chunk 592\n",
      "Processing chunk 593\n",
      "Processing chunk 594\n",
      "Processing chunk 595\n",
      "Processing chunk 596\n",
      "Processing chunk 597\n",
      "Processing chunk 598\n",
      "Processing chunk 599\n",
      "Processing chunk 600\n",
      "Processing chunk 601\n",
      "Processing chunk 602\n",
      "Processing chunk 603\n",
      "Processing chunk 604\n",
      "Processing chunk 605\n",
      "Processing chunk 606\n",
      "Processing chunk 607\n",
      "Processing chunk 608\n",
      "Processing chunk 609\n",
      "Processing chunk 610\n",
      "Processing chunk 611\n",
      "Processing chunk 612\n",
      "Processing chunk 613\n",
      "Processing chunk 614\n",
      "Processing chunk 615\n",
      "Processing chunk 616\n",
      "Processing chunk 617\n",
      "Processing chunk 618\n",
      "Processing chunk 619\n",
      "Processing chunk 620\n",
      "Processing chunk 621\n",
      "Processing chunk 622\n",
      "Processing chunk 623\n",
      "Processing chunk 624\n",
      "Processing chunk 625\n",
      "Processing chunk 626\n",
      "Processing chunk 627\n",
      "Processing chunk 628\n",
      "Processing chunk 629\n",
      "Processing chunk 630\n",
      "Processing chunk 631\n",
      "Processing chunk 632\n",
      "Processing chunk 633\n",
      "Processing chunk 634\n",
      "Processing chunk 635\n",
      "Processing chunk 636\n",
      "Processing chunk 637\n",
      "Processing chunk 638\n",
      "Processing chunk 639\n",
      "Processing chunk 640\n",
      "Processing chunk 641\n",
      "Processing chunk 642\n",
      "Processing chunk 643\n",
      "Processing chunk 644\n",
      "Processing chunk 645\n",
      "Processing chunk 646\n",
      "Processing chunk 647\n",
      "Processing chunk 648\n",
      "Processing chunk 649\n",
      "Processing chunk 650\n",
      "Processing chunk 651\n",
      "Processing chunk 652\n",
      "Processing chunk 653\n",
      "Processing chunk 654\n",
      "Processing chunk 655\n",
      "Processing chunk 656\n",
      "Processing chunk 657\n",
      "Processing chunk 658\n",
      "Processing chunk 659\n",
      "Processing chunk 660\n",
      "Processing chunk 661\n",
      "Processing chunk 662\n",
      "Processing chunk 663\n",
      "Processing chunk 664\n",
      "Processing chunk 665\n",
      "Processing chunk 666\n",
      "Processing chunk 667\n",
      "Processing chunk 668\n",
      "Processing chunk 669\n",
      "Processing chunk 670\n",
      "Processing chunk 671\n",
      "Processing chunk 672\n",
      "Processing chunk 673\n",
      "Processing chunk 674\n",
      "Processing chunk 675\n",
      "Processing chunk 676\n",
      "Processing chunk 677\n",
      "Processing chunk 678\n",
      "Processing chunk 679\n",
      "Processing chunk 680\n",
      "Processing chunk 681\n",
      "Processing chunk 682\n",
      "Processing chunk 683\n",
      "Processing chunk 684\n",
      "Processing chunk 685\n",
      "Processing chunk 686\n",
      "Processing chunk 687\n",
      "Processing chunk 688\n",
      "Processing chunk 689\n",
      "Processing chunk 690\n",
      "Processing chunk 691\n",
      "Processing chunk 692\n",
      "Processing chunk 693\n",
      "Processing chunk 694\n",
      "Processing chunk 695\n",
      "Processing chunk 696\n",
      "Processing chunk 697\n",
      "Processing chunk 698\n",
      "Processing chunk 699\n",
      "Processing chunk 700\n",
      "Processing chunk 701\n",
      "Processing chunk 702\n",
      "Processing chunk 703\n",
      "Processing chunk 704\n",
      "Processing chunk 705\n",
      "Processing chunk 706\n",
      "Processing chunk 707\n",
      "Processing chunk 708\n",
      "Processing chunk 709\n",
      "Processing chunk 710\n",
      "Processing chunk 711\n",
      "Processing chunk 712\n",
      "Processing chunk 713\n",
      "Processing chunk 714\n",
      "Processing chunk 715\n",
      "Processing chunk 716\n",
      "Processing chunk 717\n",
      "Processing chunk 718\n",
      "Processing chunk 719\n",
      "Processing chunk 720\n",
      "Processing chunk 721\n",
      "Processing chunk 722\n",
      "Processing chunk 723\n",
      "Processing chunk 724\n",
      "Processing chunk 725\n",
      "Processing chunk 726\n",
      "Processing chunk 727\n",
      "Processing chunk 728\n",
      "Processing chunk 729\n",
      "Processing chunk 730\n",
      "Processing chunk 731\n",
      "Processing chunk 732\n",
      "Processing chunk 733\n",
      "Processing chunk 734\n",
      "Processing chunk 735\n",
      "Processing chunk 736\n",
      "Processing chunk 737\n",
      "Processing chunk 738\n",
      "Processing chunk 739\n",
      "Processing chunk 740\n",
      "Processing chunk 741\n",
      "Processing chunk 742\n",
      "Processing chunk 743\n",
      "Processing chunk 744\n",
      "Processing chunk 745\n",
      "Processing chunk 746\n",
      "Processing chunk 747\n",
      "Processing chunk 748\n",
      "Processing chunk 749\n",
      "Processing chunk 750\n",
      "Processing chunk 751\n",
      "Processing chunk 752\n",
      "Processing chunk 753\n",
      "Processing chunk 754\n",
      "Processing chunk 755\n",
      "Processing chunk 756\n",
      "Processing chunk 757\n",
      "Processing chunk 758\n",
      "Processing chunk 759\n",
      "Processing chunk 760\n",
      "Processing chunk 761\n",
      "Processing chunk 762\n",
      "Processing chunk 763\n",
      "Processing chunk 764\n",
      "Processing chunk 765\n",
      "Processing chunk 766\n",
      "Processing chunk 767\n",
      "Processing chunk 768\n",
      "Processing chunk 769\n",
      "Processing chunk 770\n",
      "Processing chunk 771\n",
      "Processing chunk 772\n",
      "Processing chunk 773\n",
      "Processing chunk 774\n",
      "Processing chunk 775\n",
      "Processing chunk 776\n",
      "Processing chunk 777\n",
      "Processing chunk 778\n",
      "Processing chunk 779\n",
      "Processing chunk 780\n",
      "Processing chunk 781\n",
      "Processing chunk 782\n",
      "Processing chunk 783\n",
      "Processing chunk 784\n",
      "Processing chunk 785\n",
      "Processing chunk 786\n",
      "Processing chunk 787\n",
      "Processing chunk 788\n",
      "Processing chunk 789\n",
      "Processing chunk 790\n",
      "Processing chunk 791\n",
      "Processing chunk 792\n",
      "Processing chunk 793\n",
      "Processing chunk 794\n",
      "Processing chunk 795\n",
      "Processing chunk 796\n",
      "Processing chunk 797\n",
      "Processing chunk 798\n",
      "Processing chunk 799\n",
      "Processing chunk 800\n",
      "Processing chunk 801\n",
      "Processing chunk 802\n",
      "Processing chunk 803\n",
      "Processing chunk 804\n",
      "Processing chunk 805\n",
      "Processing chunk 806\n",
      "Processing chunk 807\n",
      "Processing chunk 808\n",
      "Processing chunk 809\n",
      "Processing chunk 810\n",
      "Processing chunk 811\n",
      "Processing chunk 812\n",
      "Processing chunk 813\n",
      "Processing chunk 814\n",
      "Processing chunk 815\n",
      "Processing chunk 816\n",
      "Processing chunk 817\n",
      "Processing chunk 818\n",
      "Processing chunk 819\n",
      "Processing chunk 820\n",
      "Processing chunk 821\n",
      "Processing chunk 822\n",
      "Processing chunk 823\n",
      "Processing chunk 824\n",
      "Processing chunk 825\n",
      "Processing chunk 826\n",
      "Processing chunk 827\n",
      "Processing chunk 828\n",
      "Processing chunk 829\n",
      "Processing chunk 830\n",
      "Processing chunk 831\n",
      "Processing chunk 832\n",
      "Processing chunk 833\n",
      "Processing chunk 834\n",
      "Processing chunk 835\n",
      "Processing chunk 836\n",
      "Processing chunk 837\n",
      "Processing chunk 838\n",
      "Processing chunk 839\n",
      "Processing chunk 840\n",
      "Processing chunk 841\n",
      "Processing chunk 842\n",
      "Processing chunk 843\n",
      "Processing chunk 844\n",
      "Processing chunk 845\n",
      "Processing chunk 846\n",
      "Processing chunk 847\n",
      "Processing chunk 848\n",
      "Processing chunk 849\n",
      "Processing chunk 850\n",
      "Processing chunk 851\n",
      "Processing chunk 852\n",
      "Processing chunk 853\n",
      "Processing chunk 854\n",
      "Processing chunk 855\n",
      "Processing chunk 856\n",
      "Processing chunk 857\n",
      "Processing chunk 858\n",
      "Processing chunk 859\n",
      "Processing chunk 860\n",
      "Processing chunk 861\n",
      "Processing chunk 862\n",
      "Processing chunk 863\n",
      "Processing chunk 864\n",
      "Processing chunk 865\n",
      "Processing chunk 866\n",
      "Processing chunk 867\n",
      "Processing chunk 868\n",
      "Processing chunk 869\n",
      "Processing chunk 870\n",
      "Processing chunk 871\n",
      "Processing chunk 872\n",
      "Processing chunk 873\n",
      "Processing chunk 874\n",
      "Processing chunk 875\n",
      "Processing chunk 876\n",
      "Processing chunk 877\n",
      "Processing chunk 878\n",
      "Processing chunk 879\n",
      "Processing chunk 880\n",
      "Processing chunk 881\n",
      "Processing chunk 882\n",
      "Processing chunk 883\n",
      "Processing chunk 884\n",
      "Processing chunk 885\n",
      "Processing chunk 886\n",
      "Processing chunk 887\n",
      "Processing chunk 888\n",
      "Processing chunk 889\n",
      "Processing chunk 890\n",
      "Processing chunk 891\n",
      "Processing chunk 892\n",
      "Processing chunk 893\n",
      "Processing chunk 894\n",
      "Processing chunk 895\n",
      "Processing chunk 896\n",
      "Processing chunk 897\n",
      "Processing chunk 898\n",
      "Processing chunk 899\n",
      "Processing chunk 900\n",
      "Processing chunk 901\n",
      "Processing chunk 902\n",
      "Processing chunk 903\n",
      "Processing chunk 904\n",
      "Processing chunk 905\n",
      "Processing chunk 906\n",
      "Processing chunk 907\n",
      "Processing chunk 908\n",
      "Processing chunk 909\n",
      "Processing chunk 910\n",
      "Processing chunk 911\n",
      "Processing chunk 912\n",
      "Processing chunk 913\n",
      "Processing chunk 914\n",
      "Processing chunk 915\n",
      "Processing chunk 916\n",
      "Processing chunk 917\n",
      "Processing chunk 918\n",
      "Processing chunk 919\n",
      "Processing chunk 920\n",
      "Processing chunk 921\n",
      "Processing chunk 922\n",
      "Processing chunk 923\n",
      "Processing chunk 924\n",
      "Processing chunk 925\n",
      "Processing chunk 926\n",
      "Processing chunk 927\n",
      "Processing chunk 928\n",
      "Processing chunk 929\n",
      "Processing chunk 930\n",
      "Processing chunk 931\n",
      "Processing chunk 932\n",
      "Processing chunk 933\n",
      "Processing chunk 934\n",
      "Processing chunk 935\n",
      "Processing chunk 936\n",
      "Processing chunk 937\n",
      "Processing chunk 938\n",
      "Processing chunk 939\n",
      "Processing chunk 940\n",
      "Processing chunk 941\n",
      "Processing chunk 942\n",
      "Processing chunk 943\n",
      "Processing chunk 944\n",
      "Processing chunk 945\n",
      "Processing chunk 946\n",
      "Processing chunk 947\n",
      "Processing chunk 948\n",
      "Processing chunk 949\n",
      "Processing chunk 950\n",
      "Processing chunk 951\n",
      "Processing chunk 952\n",
      "Processing chunk 953\n",
      "Processing chunk 954\n",
      "Processing chunk 955\n",
      "Processing chunk 956\n",
      "Processing chunk 957\n",
      "Processing chunk 958\n",
      "Processing chunk 959\n",
      "Processing chunk 960\n",
      "Processing chunk 961\n",
      "Processing chunk 962\n",
      "Processing chunk 963\n",
      "Processing chunk 964\n",
      "Processing chunk 965\n",
      "Processing chunk 966\n",
      "Processing chunk 967\n",
      "Processing chunk 968\n",
      "Processing chunk 969\n",
      "Processing chunk 970\n",
      "Processing chunk 971\n",
      "Processing chunk 972\n",
      "Processing chunk 973\n",
      "Processing chunk 974\n",
      "Processing chunk 975\n",
      "Processing chunk 976\n",
      "Processing chunk 977\n",
      "Processing chunk 978\n",
      "Processing chunk 979\n",
      "Processing chunk 980\n",
      "Processing chunk 981\n",
      "Processing chunk 982\n",
      "Processing chunk 983\n",
      "Processing chunk 984\n",
      "Processing chunk 985\n",
      "Processing chunk 986\n",
      "Processing chunk 987\n",
      "Processing chunk 988\n",
      "Processing chunk 989\n",
      "Processing chunk 990\n",
      "Processing chunk 991\n",
      "Processing chunk 992\n",
      "Processing chunk 993\n",
      "Processing chunk 994\n",
      "Processing chunk 995\n",
      "Processing chunk 996\n",
      "Processing chunk 997\n",
      "Processing chunk 998\n",
      "Processing chunk 999\n",
      "Processing chunk 1000\n",
      "Processing chunk 1001\n",
      "Processing chunk 1002\n",
      "Processing chunk 1003\n",
      "Processing chunk 1004\n",
      "Processing chunk 1005\n",
      "Processing chunk 1006\n",
      "Processing chunk 1007\n",
      "Processing chunk 1008\n",
      "Processing chunk 1009\n",
      "Processing chunk 1010\n",
      "Processing chunk 1011\n",
      "Processing chunk 1012\n",
      "Processing chunk 1013\n",
      "Processing chunk 1014\n",
      "Processing chunk 1015\n",
      "Processing chunk 1016\n",
      "Processing chunk 1017\n",
      "Processing chunk 1018\n",
      "Processing chunk 1019\n",
      "Processing chunk 1020\n",
      "Processing chunk 1021\n",
      "Processing chunk 1022\n",
      "Processing chunk 1023\n",
      "Processing chunk 1024\n",
      "Processing chunk 1025\n",
      "Processing chunk 1026\n",
      "Processing chunk 1027\n",
      "Processing chunk 1028\n",
      "Processing chunk 1029\n",
      "Processing chunk 1030\n",
      "Processing chunk 1031\n",
      "Processing chunk 1032\n",
      "Processing chunk 1033\n",
      "Processing chunk 1034\n",
      "Processing chunk 1035\n",
      "Processing chunk 1036\n",
      "Processing chunk 1037\n",
      "Processing chunk 1038\n",
      "Processing chunk 1039\n",
      "Processing chunk 1040\n",
      "Processing chunk 1041\n",
      "Processing chunk 1042\n",
      "Processing chunk 1043\n",
      "Processing chunk 1044\n",
      "Processing chunk 1045\n",
      "Processing chunk 1046\n",
      "Processing chunk 1047\n",
      "Processing chunk 1048\n",
      "Processing chunk 1049\n",
      "Processing chunk 1050\n",
      "Processing chunk 1051\n",
      "Processing chunk 1052\n",
      "Processing chunk 1053\n",
      "Processing chunk 1054\n",
      "Processing chunk 1055\n",
      "Processing chunk 1056\n",
      "Processing chunk 1057\n",
      "Processing chunk 1058\n",
      "Processing chunk 1059\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import librosa\n",
    "\n",
    "# Function to preprocess the audio files\n",
    "def preprocess_audio(audio, sr, target_sr=16000, max_length=5):\n",
    "    # Resample audio if needed\n",
    "    if sr != target_sr:\n",
    "        audio = librosa.resample(audio, orig_sr=sr, target_sr=target_sr)\n",
    "        sr = target_sr\n",
    "    \n",
    "    # Normalize the audio\n",
    "    audio = librosa.util.normalize(audio)\n",
    "    \n",
    "    # Trim leading and trailing silence\n",
    "    audio, _ = librosa.effects.trim(audio)\n",
    "    \n",
    "    # Calculate the required padding\n",
    "    target_length = target_sr * max_length\n",
    "    if len(audio) < target_length:\n",
    "        padding = target_length - len(audio)\n",
    "        # Pad the audio file\n",
    "        audio = np.pad(audio, (0, padding), 'constant')\n",
    "    else:\n",
    "        # If audio is longer than target length, truncate it\n",
    "        audio = audio[:target_length]\n",
    "    \n",
    "    return audio, sr\n",
    "\n",
    "# Function to preprocess a chunk of the DataFrame\n",
    "def preprocess_chunk(df_chunk, target_sr=16000, max_length=5):\n",
    "    processed_data = []\n",
    "    \n",
    "    for idx, row in df_chunk.iterrows():\n",
    "        file_path = row['file_path']\n",
    "        category = row['category']\n",
    "        \n",
    "        try:\n",
    "            # Load the audio file\n",
    "            audio, sr = librosa.load(file_path, sr=None)\n",
    "            \n",
    "            # Preprocess the audio\n",
    "            preprocessed_audio, preprocessed_sr = preprocess_audio(audio, sr, target_sr, max_length)\n",
    "            \n",
    "            # Append preprocessed data\n",
    "            processed_data.append({\n",
    "                'file_path': file_path,\n",
    "                'category': category,\n",
    "                'audio': preprocessed_audio,\n",
    "                'sr': preprocessed_sr\n",
    "            })\n",
    "        except Exception as e:\n",
    "            print(f\"Error processing file {file_path}: {e}\")\n",
    "    \n",
    "    return pd.DataFrame(processed_data)\n",
    "\n",
    "# Function to process the DataFrame in chunks\n",
    "def process_dataframe_in_chunks(df, chunk_size=100, target_sr=16000, max_length=5):\n",
    "    processed_chunks = []\n",
    "    \n",
    "    # Split DataFrame into chunks and process each chunk\n",
    "    for chunk_idx, df_chunk in enumerate(np.array_split(df, len(df) // chunk_size + 1)):\n",
    "        print(f\"Processing chunk {chunk_idx + 1}\")\n",
    "        processed_chunk = preprocess_chunk(df_chunk, target_sr, max_length)\n",
    "        processed_chunks.append(processed_chunk)\n",
    "        del df_chunk  # Free memory\n",
    "    \n",
    "    # Concatenate all processed chunks into a single DataFrame\n",
    "    processed_df = pd.concat(processed_chunks, ignore_index=True)\n",
    "    \n",
    "    return processed_df\n",
    "\n",
    "# Example of how to use the function\n",
    "# Assume df has the columns: file_path, category\n",
    "preprocessed_df = process_dataframe_in_chunks(df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_path</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>data\\backward\\0165e0e8_nohash_0.wav</td>\n",
       "      <td>backward</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>data\\backward\\017c4098_nohash_0.wav</td>\n",
       "      <td>backward</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>data\\backward\\017c4098_nohash_1.wav</td>\n",
       "      <td>backward</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>data\\backward\\017c4098_nohash_2.wav</td>\n",
       "      <td>backward</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>data\\backward\\017c4098_nohash_3.wav</td>\n",
       "      <td>backward</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             file_path  category\n",
       "0  data\\backward\\0165e0e8_nohash_0.wav  backward\n",
       "1  data\\backward\\017c4098_nohash_0.wav  backward\n",
       "2  data\\backward\\017c4098_nohash_1.wav  backward\n",
       "3  data\\backward\\017c4098_nohash_2.wav  backward\n",
       "4  data\\backward\\017c4098_nohash_3.wav  backward"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import librosa\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_mfcc_features(audio, sr, n_mfcc=13):\n",
    "    # Extract MFCC features\n",
    "    mfcc = librosa.feature.mfcc(y=audio, sr=sr, n_mfcc=n_mfcc)\n",
    "    # Take the mean of the MFCC coefficients over time\n",
    "    mfcc_mean = mfcc.mean(axis=1)\n",
    "    return mfcc_mean\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_audio(file_path, target_sr=16000, n_mfcc=13):\n",
    "    # Load the audio file\n",
    "    audio, sr = librosa.load(file_path, sr=None)\n",
    "    \n",
    "    # Resample and preprocess audio\n",
    "    if sr != target_sr:\n",
    "        audio = librosa.resample(audio, orig_sr=sr, target_sr=target_sr)\n",
    "        sr = target_sr\n",
    "    \n",
    "    audio = librosa.util.normalize(audio)\n",
    "    audio, _ = librosa.effects.trim(audio)\n",
    "    \n",
    "    target_length = target_sr * 5  # Max length in seconds (e.g., 5 seconds)\n",
    "    if len(audio) < target_length:\n",
    "        padding = target_length - len(audio)\n",
    "        audio = np.pad(audio, (0, padding), 'constant')\n",
    "    else:\n",
    "        audio = audio[:target_length]\n",
    "    \n",
    "    # Extract MFCC features\n",
    "    mfcc_features = extract_mfcc_features(audio, sr, n_mfcc)\n",
    "    \n",
    "    return mfcc_features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_dataframe_in_chunks(df, chunk_size=100, target_sr=16000, n_mfcc=13):\n",
    "    # Initialize lists to store processed data\n",
    "    mfcc_features_list = []\n",
    "    file_paths = []\n",
    "    \n",
    "    for chunk_idx, df_chunk in enumerate(np.array_split(df, len(df) // chunk_size + 1)):\n",
    "        print(f\"Processing chunk {chunk_idx + 1}\")\n",
    "        \n",
    "        for idx, row in df_chunk.iterrows():\n",
    "            file_path = row['file_path']\n",
    "            \n",
    "            try:\n",
    "                # Process audio file and extract MFCC features\n",
    "                mfcc_features = process_audio(file_path, target_sr, n_mfcc)\n",
    "                mfcc_features_list.append(mfcc_features)\n",
    "                file_paths.append(file_path)\n",
    "            except Exception as e:\n",
    "                print(f\"Error processing file {file_path}: {e}\")\n",
    "        \n",
    "        # Free memory\n",
    "        del df_chunk\n",
    "\n",
    "    # Convert lists to DataFrame\n",
    "    mfcc_df = pd.DataFrame(mfcc_features_list, columns=[f'mfcc_{i}' for i in range(n_mfcc)])\n",
    "    result_df = pd.DataFrame({'file_path': file_paths}).join(mfcc_df)\n",
    "\n",
    "    return result_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\HP\\anaconda3\\envs\\py310\\lib\\site-packages\\numpy\\core\\fromnumeric.py:59: FutureWarning: 'DataFrame.swapaxes' is deprecated and will be removed in a future version. Please use 'DataFrame.transpose' instead.\n",
      "  return bound(*args, **kwds)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing chunk 1\n",
      "Processing chunk 2\n",
      "Processing chunk 3\n",
      "Processing chunk 4\n",
      "Processing chunk 5\n",
      "Processing chunk 6\n",
      "Processing chunk 7\n",
      "Processing chunk 8\n",
      "Processing chunk 9\n",
      "Processing chunk 10\n",
      "Processing chunk 11\n",
      "Processing chunk 12\n",
      "Processing chunk 13\n",
      "Processing chunk 14\n",
      "Processing chunk 15\n",
      "Processing chunk 16\n",
      "Processing chunk 17\n",
      "Processing chunk 18\n",
      "Processing chunk 19\n",
      "Processing chunk 20\n",
      "Processing chunk 21\n",
      "Processing chunk 22\n",
      "Processing chunk 23\n",
      "Processing chunk 24\n",
      "Processing chunk 25\n",
      "Processing chunk 26\n",
      "Processing chunk 27\n",
      "Processing chunk 28\n",
      "Processing chunk 29\n",
      "Processing chunk 30\n",
      "Processing chunk 31\n",
      "Processing chunk 32\n",
      "Processing chunk 33\n",
      "Processing chunk 34\n",
      "Processing chunk 35\n",
      "Processing chunk 36\n",
      "Processing chunk 37\n",
      "Processing chunk 38\n",
      "Processing chunk 39\n",
      "Processing chunk 40\n",
      "Processing chunk 41\n",
      "Processing chunk 42\n",
      "Processing chunk 43\n",
      "Processing chunk 44\n",
      "Processing chunk 45\n",
      "Processing chunk 46\n",
      "Processing chunk 47\n",
      "Processing chunk 48\n",
      "Processing chunk 49\n",
      "Processing chunk 50\n",
      "Processing chunk 51\n",
      "Processing chunk 52\n",
      "Processing chunk 53\n",
      "Processing chunk 54\n",
      "Processing chunk 55\n",
      "Processing chunk 56\n",
      "Processing chunk 57\n",
      "Processing chunk 58\n",
      "Processing chunk 59\n",
      "Processing chunk 60\n",
      "Processing chunk 61\n",
      "Processing chunk 62\n",
      "Processing chunk 63\n",
      "Processing chunk 64\n",
      "Processing chunk 65\n",
      "Processing chunk 66\n",
      "Processing chunk 67\n",
      "Processing chunk 68\n",
      "Processing chunk 69\n",
      "Processing chunk 70\n",
      "Processing chunk 71\n",
      "Processing chunk 72\n",
      "Processing chunk 73\n",
      "Processing chunk 74\n",
      "Processing chunk 75\n",
      "Processing chunk 76\n",
      "Processing chunk 77\n",
      "Processing chunk 78\n",
      "Processing chunk 79\n",
      "Processing chunk 80\n",
      "Processing chunk 81\n",
      "Processing chunk 82\n",
      "Processing chunk 83\n",
      "Processing chunk 84\n",
      "Processing chunk 85\n",
      "Processing chunk 86\n",
      "Processing chunk 87\n",
      "Processing chunk 88\n",
      "Processing chunk 89\n",
      "Processing chunk 90\n",
      "Processing chunk 91\n",
      "Processing chunk 92\n",
      "Processing chunk 93\n",
      "Processing chunk 94\n",
      "Processing chunk 95\n",
      "Processing chunk 96\n",
      "Processing chunk 97\n",
      "Processing chunk 98\n",
      "Processing chunk 99\n",
      "Processing chunk 100\n",
      "Processing chunk 101\n",
      "Processing chunk 102\n",
      "Processing chunk 103\n",
      "Processing chunk 104\n",
      "Processing chunk 105\n",
      "Processing chunk 106\n",
      "Processing chunk 107\n",
      "Processing chunk 108\n",
      "Processing chunk 109\n",
      "Processing chunk 110\n",
      "Processing chunk 111\n",
      "Processing chunk 112\n",
      "Processing chunk 113\n",
      "Processing chunk 114\n",
      "Processing chunk 115\n",
      "Processing chunk 116\n",
      "Processing chunk 117\n",
      "Processing chunk 118\n",
      "Processing chunk 119\n",
      "Processing chunk 120\n",
      "Processing chunk 121\n",
      "Processing chunk 122\n",
      "Processing chunk 123\n",
      "Processing chunk 124\n",
      "Processing chunk 125\n",
      "Processing chunk 126\n",
      "Processing chunk 127\n",
      "Processing chunk 128\n",
      "Processing chunk 129\n",
      "Processing chunk 130\n",
      "Processing chunk 131\n",
      "Processing chunk 132\n",
      "Processing chunk 133\n",
      "Processing chunk 134\n",
      "Processing chunk 135\n",
      "Processing chunk 136\n",
      "Processing chunk 137\n",
      "Processing chunk 138\n",
      "Processing chunk 139\n",
      "Processing chunk 140\n",
      "Processing chunk 141\n",
      "Processing chunk 142\n",
      "Processing chunk 143\n",
      "Processing chunk 144\n",
      "Processing chunk 145\n",
      "Processing chunk 146\n",
      "Processing chunk 147\n",
      "Processing chunk 148\n",
      "Processing chunk 149\n",
      "Processing chunk 150\n",
      "Processing chunk 151\n",
      "Processing chunk 152\n",
      "Processing chunk 153\n",
      "Processing chunk 154\n",
      "Processing chunk 155\n",
      "Processing chunk 156\n",
      "Processing chunk 157\n",
      "Processing chunk 158\n",
      "Processing chunk 159\n",
      "Processing chunk 160\n",
      "Processing chunk 161\n",
      "Processing chunk 162\n",
      "Processing chunk 163\n",
      "Processing chunk 164\n",
      "Processing chunk 165\n",
      "Processing chunk 166\n",
      "Processing chunk 167\n",
      "Processing chunk 168\n",
      "Processing chunk 169\n",
      "Processing chunk 170\n",
      "Processing chunk 171\n",
      "Processing chunk 172\n",
      "Processing chunk 173\n",
      "Processing chunk 174\n",
      "Processing chunk 175\n",
      "Processing chunk 176\n",
      "Processing chunk 177\n",
      "Processing chunk 178\n",
      "Processing chunk 179\n",
      "Processing chunk 180\n",
      "Processing chunk 181\n",
      "Processing chunk 182\n",
      "Processing chunk 183\n",
      "Processing chunk 184\n",
      "Processing chunk 185\n",
      "Processing chunk 186\n",
      "Processing chunk 187\n",
      "Processing chunk 188\n",
      "Processing chunk 189\n",
      "Processing chunk 190\n",
      "Processing chunk 191\n",
      "Processing chunk 192\n",
      "Processing chunk 193\n",
      "Processing chunk 194\n",
      "Processing chunk 195\n",
      "Processing chunk 196\n",
      "Processing chunk 197\n",
      "Processing chunk 198\n",
      "Processing chunk 199\n",
      "Processing chunk 200\n",
      "Processing chunk 201\n",
      "Processing chunk 202\n",
      "Processing chunk 203\n",
      "Processing chunk 204\n",
      "Processing chunk 205\n",
      "Processing chunk 206\n",
      "Processing chunk 207\n",
      "Processing chunk 208\n",
      "Processing chunk 209\n",
      "Processing chunk 210\n",
      "Processing chunk 211\n",
      "Processing chunk 212\n",
      "Processing chunk 213\n",
      "Processing chunk 214\n",
      "Processing chunk 215\n",
      "Processing chunk 216\n",
      "Processing chunk 217\n",
      "Processing chunk 218\n",
      "Processing chunk 219\n",
      "Processing chunk 220\n",
      "Processing chunk 221\n",
      "Processing chunk 222\n",
      "Processing chunk 223\n",
      "Processing chunk 224\n",
      "Processing chunk 225\n",
      "Processing chunk 226\n",
      "Processing chunk 227\n",
      "Processing chunk 228\n",
      "Processing chunk 229\n",
      "Processing chunk 230\n",
      "Processing chunk 231\n",
      "Processing chunk 232\n",
      "Processing chunk 233\n",
      "Processing chunk 234\n",
      "Processing chunk 235\n",
      "Processing chunk 236\n",
      "Processing chunk 237\n",
      "Processing chunk 238\n",
      "Processing chunk 239\n",
      "Processing chunk 240\n",
      "Processing chunk 241\n",
      "Processing chunk 242\n",
      "Processing chunk 243\n",
      "Processing chunk 244\n",
      "Processing chunk 245\n",
      "Processing chunk 246\n",
      "Processing chunk 247\n",
      "Processing chunk 248\n",
      "Processing chunk 249\n",
      "Processing chunk 250\n",
      "Processing chunk 251\n",
      "Processing chunk 252\n",
      "Processing chunk 253\n",
      "Processing chunk 254\n",
      "Processing chunk 255\n",
      "Processing chunk 256\n",
      "Processing chunk 257\n",
      "Processing chunk 258\n",
      "Processing chunk 259\n",
      "Processing chunk 260\n",
      "Processing chunk 261\n",
      "Processing chunk 262\n",
      "Processing chunk 263\n",
      "Processing chunk 264\n",
      "Processing chunk 265\n",
      "Processing chunk 266\n",
      "Processing chunk 267\n",
      "Processing chunk 268\n",
      "Processing chunk 269\n",
      "Processing chunk 270\n",
      "Processing chunk 271\n",
      "Processing chunk 272\n",
      "Processing chunk 273\n",
      "Processing chunk 274\n",
      "Processing chunk 275\n",
      "Processing chunk 276\n",
      "Processing chunk 277\n",
      "Processing chunk 278\n",
      "Processing chunk 279\n",
      "Processing chunk 280\n",
      "Processing chunk 281\n",
      "Processing chunk 282\n",
      "Processing chunk 283\n",
      "Processing chunk 284\n",
      "Processing chunk 285\n",
      "Processing chunk 286\n",
      "Processing chunk 287\n",
      "Processing chunk 288\n",
      "Processing chunk 289\n",
      "Processing chunk 290\n",
      "Processing chunk 291\n",
      "Processing chunk 292\n",
      "Processing chunk 293\n",
      "Processing chunk 294\n",
      "Processing chunk 295\n",
      "Processing chunk 296\n",
      "Processing chunk 297\n",
      "Processing chunk 298\n",
      "Processing chunk 299\n",
      "Processing chunk 300\n",
      "Processing chunk 301\n",
      "Processing chunk 302\n",
      "Processing chunk 303\n",
      "Processing chunk 304\n",
      "Processing chunk 305\n",
      "Processing chunk 306\n",
      "Processing chunk 307\n",
      "Processing chunk 308\n",
      "Processing chunk 309\n",
      "Processing chunk 310\n",
      "Processing chunk 311\n",
      "Processing chunk 312\n",
      "Processing chunk 313\n",
      "Processing chunk 314\n",
      "Processing chunk 315\n",
      "Processing chunk 316\n",
      "Processing chunk 317\n",
      "Processing chunk 318\n",
      "Processing chunk 319\n",
      "Processing chunk 320\n",
      "Processing chunk 321\n",
      "Processing chunk 322\n",
      "Processing chunk 323\n",
      "Processing chunk 324\n",
      "Processing chunk 325\n",
      "Processing chunk 326\n",
      "Processing chunk 327\n",
      "Processing chunk 328\n",
      "Processing chunk 329\n",
      "Processing chunk 330\n",
      "Processing chunk 331\n",
      "Processing chunk 332\n",
      "Processing chunk 333\n",
      "Processing chunk 334\n",
      "Processing chunk 335\n",
      "Processing chunk 336\n",
      "Processing chunk 337\n",
      "Processing chunk 338\n",
      "Processing chunk 339\n",
      "Processing chunk 340\n",
      "Processing chunk 341\n",
      "Processing chunk 342\n",
      "Processing chunk 343\n",
      "Processing chunk 344\n",
      "Processing chunk 345\n",
      "Processing chunk 346\n",
      "Processing chunk 347\n",
      "Processing chunk 348\n",
      "Processing chunk 349\n",
      "Processing chunk 350\n",
      "Processing chunk 351\n",
      "Processing chunk 352\n",
      "Processing chunk 353\n",
      "Processing chunk 354\n",
      "Processing chunk 355\n",
      "Processing chunk 356\n",
      "Processing chunk 357\n",
      "Processing chunk 358\n",
      "Processing chunk 359\n",
      "Processing chunk 360\n",
      "Processing chunk 361\n",
      "Processing chunk 362\n",
      "Processing chunk 363\n",
      "Processing chunk 364\n",
      "Processing chunk 365\n",
      "Processing chunk 366\n",
      "Processing chunk 367\n",
      "Processing chunk 368\n",
      "Processing chunk 369\n",
      "Processing chunk 370\n",
      "Processing chunk 371\n",
      "Processing chunk 372\n",
      "Processing chunk 373\n",
      "Processing chunk 374\n",
      "Processing chunk 375\n",
      "Processing chunk 376\n",
      "Processing chunk 377\n",
      "Processing chunk 378\n",
      "Processing chunk 379\n",
      "Processing chunk 380\n",
      "Processing chunk 381\n",
      "Processing chunk 382\n",
      "Processing chunk 383\n",
      "Processing chunk 384\n",
      "Processing chunk 385\n",
      "Processing chunk 386\n",
      "Processing chunk 387\n",
      "Processing chunk 388\n",
      "Processing chunk 389\n",
      "Processing chunk 390\n",
      "Processing chunk 391\n",
      "Processing chunk 392\n",
      "Processing chunk 393\n",
      "Processing chunk 394\n",
      "Processing chunk 395\n",
      "Processing chunk 396\n",
      "Processing chunk 397\n",
      "Processing chunk 398\n",
      "Processing chunk 399\n",
      "Processing chunk 400\n",
      "Processing chunk 401\n",
      "Processing chunk 402\n",
      "Processing chunk 403\n",
      "Processing chunk 404\n",
      "Processing chunk 405\n",
      "Processing chunk 406\n",
      "Processing chunk 407\n",
      "Processing chunk 408\n",
      "Processing chunk 409\n",
      "Processing chunk 410\n",
      "Processing chunk 411\n",
      "Processing chunk 412\n",
      "Processing chunk 413\n",
      "Processing chunk 414\n",
      "Processing chunk 415\n",
      "Processing chunk 416\n",
      "Processing chunk 417\n",
      "Processing chunk 418\n",
      "Processing chunk 419\n",
      "Processing chunk 420\n",
      "Processing chunk 421\n",
      "Processing chunk 422\n",
      "Processing chunk 423\n",
      "Processing chunk 424\n",
      "Processing chunk 425\n",
      "Processing chunk 426\n",
      "Processing chunk 427\n",
      "Processing chunk 428\n",
      "Processing chunk 429\n",
      "Processing chunk 430\n",
      "Processing chunk 431\n",
      "Processing chunk 432\n",
      "Processing chunk 433\n",
      "Processing chunk 434\n",
      "Processing chunk 435\n",
      "Processing chunk 436\n",
      "Processing chunk 437\n",
      "Processing chunk 438\n",
      "Processing chunk 439\n",
      "Processing chunk 440\n",
      "Processing chunk 441\n",
      "Processing chunk 442\n",
      "Processing chunk 443\n",
      "Processing chunk 444\n",
      "Processing chunk 445\n",
      "Processing chunk 446\n",
      "Processing chunk 447\n",
      "Processing chunk 448\n",
      "Processing chunk 449\n",
      "Processing chunk 450\n",
      "Processing chunk 451\n",
      "Processing chunk 452\n",
      "Processing chunk 453\n",
      "Processing chunk 454\n",
      "Processing chunk 455\n",
      "Processing chunk 456\n",
      "Processing chunk 457\n",
      "Processing chunk 458\n",
      "Processing chunk 459\n",
      "Processing chunk 460\n",
      "Processing chunk 461\n",
      "Processing chunk 462\n",
      "Processing chunk 463\n",
      "Processing chunk 464\n",
      "Processing chunk 465\n",
      "Processing chunk 466\n",
      "Processing chunk 467\n",
      "Processing chunk 468\n",
      "Processing chunk 469\n",
      "Processing chunk 470\n",
      "Processing chunk 471\n",
      "Processing chunk 472\n",
      "Processing chunk 473\n",
      "Processing chunk 474\n",
      "Processing chunk 475\n",
      "Processing chunk 476\n",
      "Processing chunk 477\n",
      "Processing chunk 478\n",
      "Processing chunk 479\n",
      "Processing chunk 480\n",
      "Processing chunk 481\n",
      "Processing chunk 482\n",
      "Processing chunk 483\n",
      "Processing chunk 484\n",
      "Processing chunk 485\n",
      "Processing chunk 486\n",
      "Processing chunk 487\n",
      "Processing chunk 488\n",
      "Processing chunk 489\n",
      "Processing chunk 490\n",
      "Processing chunk 491\n",
      "Processing chunk 492\n",
      "Processing chunk 493\n",
      "Processing chunk 494\n",
      "Processing chunk 495\n",
      "Processing chunk 496\n",
      "Processing chunk 497\n",
      "Processing chunk 498\n",
      "Processing chunk 499\n",
      "Processing chunk 500\n",
      "Processing chunk 501\n",
      "Processing chunk 502\n",
      "Processing chunk 503\n",
      "Processing chunk 504\n",
      "Processing chunk 505\n",
      "Processing chunk 506\n",
      "Processing chunk 507\n",
      "Processing chunk 508\n",
      "Processing chunk 509\n",
      "Processing chunk 510\n",
      "Processing chunk 511\n",
      "Processing chunk 512\n",
      "Processing chunk 513\n",
      "Processing chunk 514\n",
      "Processing chunk 515\n",
      "Processing chunk 516\n",
      "Processing chunk 517\n",
      "Processing chunk 518\n",
      "Processing chunk 519\n",
      "Processing chunk 520\n",
      "Processing chunk 521\n",
      "Processing chunk 522\n",
      "Processing chunk 523\n",
      "Processing chunk 524\n",
      "Processing chunk 525\n",
      "Processing chunk 526\n",
      "Processing chunk 527\n",
      "Processing chunk 528\n",
      "Processing chunk 529\n",
      "Processing chunk 530\n",
      "Processing chunk 531\n",
      "Processing chunk 532\n",
      "Processing chunk 533\n",
      "Processing chunk 534\n",
      "Processing chunk 535\n",
      "Processing chunk 536\n",
      "Processing chunk 537\n",
      "Processing chunk 538\n",
      "Processing chunk 539\n",
      "Processing chunk 540\n",
      "Processing chunk 541\n",
      "Processing chunk 542\n",
      "Processing chunk 543\n",
      "Processing chunk 544\n",
      "Processing chunk 545\n",
      "Processing chunk 546\n",
      "Processing chunk 547\n",
      "Processing chunk 548\n",
      "Processing chunk 549\n",
      "Processing chunk 550\n",
      "Processing chunk 551\n",
      "Processing chunk 552\n",
      "Processing chunk 553\n",
      "Processing chunk 554\n",
      "Processing chunk 555\n",
      "Processing chunk 556\n",
      "Processing chunk 557\n",
      "Processing chunk 558\n",
      "Processing chunk 559\n",
      "Processing chunk 560\n",
      "Processing chunk 561\n",
      "Processing chunk 562\n",
      "Processing chunk 563\n",
      "Processing chunk 564\n",
      "Processing chunk 565\n",
      "Processing chunk 566\n",
      "Processing chunk 567\n",
      "Processing chunk 568\n",
      "Processing chunk 569\n",
      "Processing chunk 570\n",
      "Processing chunk 571\n",
      "Processing chunk 572\n",
      "Processing chunk 573\n",
      "Processing chunk 574\n",
      "Processing chunk 575\n",
      "Processing chunk 576\n",
      "Processing chunk 577\n",
      "Processing chunk 578\n",
      "Processing chunk 579\n",
      "Processing chunk 580\n",
      "Processing chunk 581\n",
      "Processing chunk 582\n",
      "Processing chunk 583\n",
      "Processing chunk 584\n",
      "Processing chunk 585\n",
      "Processing chunk 586\n",
      "Processing chunk 587\n",
      "Processing chunk 588\n",
      "Processing chunk 589\n",
      "Processing chunk 590\n",
      "Processing chunk 591\n",
      "Processing chunk 592\n",
      "Processing chunk 593\n",
      "Processing chunk 594\n",
      "Processing chunk 595\n",
      "Processing chunk 596\n",
      "Processing chunk 597\n",
      "Processing chunk 598\n",
      "Processing chunk 599\n",
      "Processing chunk 600\n",
      "Processing chunk 601\n",
      "Processing chunk 602\n",
      "Processing chunk 603\n",
      "Processing chunk 604\n",
      "Processing chunk 605\n",
      "Processing chunk 606\n",
      "Processing chunk 607\n",
      "Processing chunk 608\n",
      "Processing chunk 609\n",
      "Processing chunk 610\n",
      "Processing chunk 611\n",
      "Processing chunk 612\n",
      "Processing chunk 613\n",
      "Processing chunk 614\n",
      "Processing chunk 615\n",
      "Processing chunk 616\n",
      "Processing chunk 617\n",
      "Processing chunk 618\n",
      "Processing chunk 619\n",
      "Processing chunk 620\n",
      "Processing chunk 621\n",
      "Processing chunk 622\n",
      "Processing chunk 623\n",
      "Processing chunk 624\n",
      "Processing chunk 625\n",
      "Processing chunk 626\n",
      "Processing chunk 627\n",
      "Processing chunk 628\n",
      "Processing chunk 629\n",
      "Processing chunk 630\n",
      "Processing chunk 631\n",
      "Processing chunk 632\n",
      "Processing chunk 633\n",
      "Processing chunk 634\n",
      "Processing chunk 635\n",
      "Processing chunk 636\n",
      "Processing chunk 637\n",
      "Processing chunk 638\n",
      "Processing chunk 639\n",
      "Processing chunk 640\n",
      "Processing chunk 641\n",
      "Processing chunk 642\n",
      "Processing chunk 643\n",
      "Processing chunk 644\n",
      "Processing chunk 645\n",
      "Processing chunk 646\n",
      "Processing chunk 647\n",
      "Processing chunk 648\n",
      "Processing chunk 649\n",
      "Processing chunk 650\n",
      "Processing chunk 651\n",
      "Processing chunk 652\n",
      "Processing chunk 653\n",
      "Processing chunk 654\n",
      "Processing chunk 655\n",
      "Processing chunk 656\n",
      "Processing chunk 657\n",
      "Processing chunk 658\n",
      "Processing chunk 659\n",
      "Processing chunk 660\n",
      "Processing chunk 661\n",
      "Processing chunk 662\n",
      "Processing chunk 663\n",
      "Processing chunk 664\n",
      "Processing chunk 665\n",
      "Processing chunk 666\n",
      "Processing chunk 667\n",
      "Processing chunk 668\n",
      "Processing chunk 669\n",
      "Processing chunk 670\n",
      "Processing chunk 671\n",
      "Processing chunk 672\n",
      "Processing chunk 673\n",
      "Processing chunk 674\n",
      "Processing chunk 675\n",
      "Processing chunk 676\n",
      "Processing chunk 677\n",
      "Processing chunk 678\n",
      "Processing chunk 679\n",
      "Processing chunk 680\n",
      "Processing chunk 681\n",
      "Processing chunk 682\n",
      "Processing chunk 683\n",
      "Processing chunk 684\n",
      "Processing chunk 685\n",
      "Processing chunk 686\n",
      "Processing chunk 687\n",
      "Processing chunk 688\n",
      "Processing chunk 689\n",
      "Processing chunk 690\n",
      "Processing chunk 691\n",
      "Processing chunk 692\n",
      "Processing chunk 693\n",
      "Processing chunk 694\n",
      "Processing chunk 695\n",
      "Processing chunk 696\n",
      "Processing chunk 697\n",
      "Processing chunk 698\n",
      "Processing chunk 699\n",
      "Processing chunk 700\n",
      "Processing chunk 701\n",
      "Processing chunk 702\n",
      "Processing chunk 703\n",
      "Processing chunk 704\n",
      "Processing chunk 705\n",
      "Processing chunk 706\n",
      "Processing chunk 707\n",
      "Processing chunk 708\n",
      "Processing chunk 709\n",
      "Processing chunk 710\n",
      "Processing chunk 711\n",
      "Processing chunk 712\n",
      "Processing chunk 713\n",
      "Processing chunk 714\n",
      "Processing chunk 715\n",
      "Processing chunk 716\n",
      "Processing chunk 717\n",
      "Processing chunk 718\n",
      "Processing chunk 719\n",
      "Processing chunk 720\n",
      "Processing chunk 721\n",
      "Processing chunk 722\n",
      "Processing chunk 723\n",
      "Processing chunk 724\n",
      "Processing chunk 725\n",
      "Processing chunk 726\n",
      "Processing chunk 727\n",
      "Processing chunk 728\n",
      "Processing chunk 729\n",
      "Processing chunk 730\n",
      "Processing chunk 731\n",
      "Processing chunk 732\n",
      "Processing chunk 733\n",
      "Processing chunk 734\n",
      "Processing chunk 735\n",
      "Processing chunk 736\n",
      "Processing chunk 737\n",
      "Processing chunk 738\n",
      "Processing chunk 739\n",
      "Processing chunk 740\n",
      "Processing chunk 741\n",
      "Processing chunk 742\n",
      "Processing chunk 743\n",
      "Processing chunk 744\n",
      "Processing chunk 745\n",
      "Processing chunk 746\n",
      "Processing chunk 747\n",
      "Processing chunk 748\n",
      "Processing chunk 749\n",
      "Processing chunk 750\n",
      "Processing chunk 751\n",
      "Processing chunk 752\n",
      "Processing chunk 753\n",
      "Processing chunk 754\n",
      "Processing chunk 755\n",
      "Processing chunk 756\n",
      "Processing chunk 757\n",
      "Processing chunk 758\n",
      "Processing chunk 759\n",
      "Processing chunk 760\n",
      "Processing chunk 761\n",
      "Processing chunk 762\n",
      "Processing chunk 763\n",
      "Processing chunk 764\n",
      "Processing chunk 765\n",
      "Processing chunk 766\n",
      "Processing chunk 767\n",
      "Processing chunk 768\n",
      "Processing chunk 769\n",
      "Processing chunk 770\n",
      "Processing chunk 771\n",
      "Processing chunk 772\n",
      "Processing chunk 773\n",
      "Processing chunk 774\n",
      "Processing chunk 775\n",
      "Processing chunk 776\n",
      "Processing chunk 777\n",
      "Processing chunk 778\n",
      "Processing chunk 779\n",
      "Processing chunk 780\n",
      "Processing chunk 781\n",
      "Processing chunk 782\n",
      "Processing chunk 783\n",
      "Processing chunk 784\n",
      "Processing chunk 785\n",
      "Processing chunk 786\n",
      "Processing chunk 787\n",
      "Processing chunk 788\n",
      "Processing chunk 789\n",
      "Processing chunk 790\n",
      "Processing chunk 791\n",
      "Processing chunk 792\n",
      "Processing chunk 793\n",
      "Processing chunk 794\n",
      "Processing chunk 795\n",
      "Processing chunk 796\n",
      "Processing chunk 797\n",
      "Processing chunk 798\n",
      "Processing chunk 799\n",
      "Processing chunk 800\n",
      "Processing chunk 801\n",
      "Processing chunk 802\n",
      "Processing chunk 803\n",
      "Processing chunk 804\n",
      "Processing chunk 805\n",
      "Processing chunk 806\n",
      "Processing chunk 807\n",
      "Processing chunk 808\n",
      "Processing chunk 809\n",
      "Processing chunk 810\n",
      "Processing chunk 811\n",
      "Processing chunk 812\n",
      "Processing chunk 813\n",
      "Processing chunk 814\n",
      "Processing chunk 815\n",
      "Processing chunk 816\n",
      "Processing chunk 817\n",
      "Processing chunk 818\n",
      "Processing chunk 819\n",
      "Processing chunk 820\n",
      "Processing chunk 821\n",
      "Processing chunk 822\n",
      "Processing chunk 823\n",
      "Processing chunk 824\n",
      "Processing chunk 825\n",
      "Processing chunk 826\n",
      "Processing chunk 827\n",
      "Processing chunk 828\n",
      "Processing chunk 829\n",
      "Processing chunk 830\n",
      "Processing chunk 831\n",
      "Processing chunk 832\n",
      "Processing chunk 833\n",
      "Processing chunk 834\n",
      "Processing chunk 835\n",
      "Processing chunk 836\n",
      "Processing chunk 837\n",
      "Processing chunk 838\n",
      "Processing chunk 839\n",
      "Processing chunk 840\n",
      "Processing chunk 841\n",
      "Processing chunk 842\n",
      "Processing chunk 843\n",
      "Processing chunk 844\n",
      "Processing chunk 845\n",
      "Processing chunk 846\n",
      "Processing chunk 847\n",
      "Processing chunk 848\n",
      "Processing chunk 849\n",
      "Processing chunk 850\n",
      "Processing chunk 851\n",
      "Processing chunk 852\n",
      "Processing chunk 853\n",
      "Processing chunk 854\n",
      "Processing chunk 855\n",
      "Processing chunk 856\n",
      "Processing chunk 857\n",
      "Processing chunk 858\n",
      "Processing chunk 859\n",
      "Processing chunk 860\n",
      "Processing chunk 861\n",
      "Processing chunk 862\n",
      "Processing chunk 863\n",
      "Processing chunk 864\n",
      "Processing chunk 865\n",
      "Processing chunk 866\n",
      "Processing chunk 867\n",
      "Processing chunk 868\n",
      "Processing chunk 869\n",
      "Processing chunk 870\n",
      "Processing chunk 871\n",
      "Processing chunk 872\n",
      "Processing chunk 873\n",
      "Processing chunk 874\n",
      "Processing chunk 875\n",
      "Processing chunk 876\n",
      "Processing chunk 877\n",
      "Processing chunk 878\n",
      "Processing chunk 879\n",
      "Processing chunk 880\n",
      "Processing chunk 881\n",
      "Processing chunk 882\n",
      "Processing chunk 883\n",
      "Processing chunk 884\n",
      "Processing chunk 885\n",
      "Processing chunk 886\n",
      "Processing chunk 887\n",
      "Processing chunk 888\n",
      "Processing chunk 889\n",
      "Processing chunk 890\n",
      "Processing chunk 891\n",
      "Processing chunk 892\n",
      "Processing chunk 893\n",
      "Processing chunk 894\n",
      "Processing chunk 895\n",
      "Processing chunk 896\n",
      "Processing chunk 897\n",
      "Processing chunk 898\n",
      "Processing chunk 899\n",
      "Processing chunk 900\n",
      "Processing chunk 901\n",
      "Processing chunk 902\n",
      "Processing chunk 903\n",
      "Processing chunk 904\n",
      "Processing chunk 905\n",
      "Processing chunk 906\n",
      "Processing chunk 907\n",
      "Processing chunk 908\n",
      "Processing chunk 909\n",
      "Processing chunk 910\n",
      "Processing chunk 911\n",
      "Processing chunk 912\n",
      "Processing chunk 913\n",
      "Processing chunk 914\n",
      "Processing chunk 915\n",
      "Processing chunk 916\n",
      "Processing chunk 917\n",
      "Processing chunk 918\n",
      "Processing chunk 919\n",
      "Processing chunk 920\n",
      "Processing chunk 921\n",
      "Processing chunk 922\n",
      "Processing chunk 923\n",
      "Processing chunk 924\n",
      "Processing chunk 925\n",
      "Processing chunk 926\n",
      "Processing chunk 927\n",
      "Processing chunk 928\n",
      "Processing chunk 929\n",
      "Processing chunk 930\n",
      "Processing chunk 931\n",
      "Processing chunk 932\n",
      "Processing chunk 933\n",
      "Processing chunk 934\n",
      "Processing chunk 935\n",
      "Processing chunk 936\n",
      "Processing chunk 937\n",
      "Processing chunk 938\n",
      "Processing chunk 939\n",
      "Processing chunk 940\n",
      "Processing chunk 941\n",
      "Processing chunk 942\n",
      "Processing chunk 943\n",
      "Processing chunk 944\n",
      "Processing chunk 945\n",
      "Processing chunk 946\n",
      "Processing chunk 947\n",
      "Processing chunk 948\n",
      "Processing chunk 949\n",
      "Processing chunk 950\n",
      "Processing chunk 951\n",
      "Processing chunk 952\n",
      "Processing chunk 953\n",
      "Processing chunk 954\n",
      "Processing chunk 955\n",
      "Processing chunk 956\n",
      "Processing chunk 957\n",
      "Processing chunk 958\n",
      "Processing chunk 959\n",
      "Processing chunk 960\n",
      "Processing chunk 961\n",
      "Processing chunk 962\n",
      "Processing chunk 963\n",
      "Processing chunk 964\n",
      "Processing chunk 965\n",
      "Processing chunk 966\n",
      "Processing chunk 967\n",
      "Processing chunk 968\n",
      "Processing chunk 969\n",
      "Processing chunk 970\n",
      "Processing chunk 971\n",
      "Processing chunk 972\n",
      "Processing chunk 973\n",
      "Processing chunk 974\n",
      "Processing chunk 975\n",
      "Processing chunk 976\n",
      "Processing chunk 977\n",
      "Processing chunk 978\n",
      "Processing chunk 979\n",
      "Processing chunk 980\n",
      "Processing chunk 981\n",
      "Processing chunk 982\n",
      "Processing chunk 983\n",
      "Processing chunk 984\n",
      "Processing chunk 985\n",
      "Processing chunk 986\n",
      "Processing chunk 987\n",
      "Processing chunk 988\n",
      "Processing chunk 989\n",
      "Processing chunk 990\n",
      "Processing chunk 991\n",
      "Processing chunk 992\n",
      "Processing chunk 993\n",
      "Processing chunk 994\n",
      "Processing chunk 995\n",
      "Processing chunk 996\n",
      "Processing chunk 997\n",
      "Processing chunk 998\n",
      "Processing chunk 999\n",
      "Processing chunk 1000\n",
      "Processing chunk 1001\n",
      "Processing chunk 1002\n",
      "Processing chunk 1003\n",
      "Processing chunk 1004\n",
      "Processing chunk 1005\n",
      "Processing chunk 1006\n",
      "Processing chunk 1007\n",
      "Processing chunk 1008\n",
      "Processing chunk 1009\n",
      "Processing chunk 1010\n",
      "Processing chunk 1011\n",
      "Processing chunk 1012\n",
      "Processing chunk 1013\n",
      "Processing chunk 1014\n",
      "Processing chunk 1015\n",
      "Processing chunk 1016\n",
      "Processing chunk 1017\n",
      "Processing chunk 1018\n",
      "Processing chunk 1019\n",
      "Processing chunk 1020\n",
      "Processing chunk 1021\n",
      "Processing chunk 1022\n",
      "Processing chunk 1023\n",
      "Processing chunk 1024\n",
      "Processing chunk 1025\n",
      "Processing chunk 1026\n",
      "Processing chunk 1027\n",
      "Processing chunk 1028\n",
      "Processing chunk 1029\n",
      "Processing chunk 1030\n",
      "Processing chunk 1031\n",
      "Processing chunk 1032\n",
      "Processing chunk 1033\n",
      "Processing chunk 1034\n",
      "Processing chunk 1035\n",
      "Processing chunk 1036\n",
      "Processing chunk 1037\n",
      "Processing chunk 1038\n",
      "Processing chunk 1039\n",
      "Processing chunk 1040\n",
      "Processing chunk 1041\n",
      "Processing chunk 1042\n",
      "Processing chunk 1043\n",
      "Processing chunk 1044\n",
      "Processing chunk 1045\n",
      "Processing chunk 1046\n",
      "Processing chunk 1047\n",
      "Processing chunk 1048\n",
      "Processing chunk 1049\n",
      "Processing chunk 1050\n",
      "Processing chunk 1051\n",
      "Processing chunk 1052\n",
      "Processing chunk 1053\n",
      "Processing chunk 1054\n",
      "Processing chunk 1055\n",
      "Processing chunk 1056\n",
      "Processing chunk 1057\n",
      "Processing chunk 1058\n",
      "Processing chunk 1059\n"
     ]
    }
   ],
   "source": [
    "# Example usage\n",
    "# Assume df has the columns: file_path, category\n",
    "mfcc_df = process_dataframe_in_chunks(df)\n",
    "\n",
    "# Merge MFCC features with the original DataFrame\n",
    "df = df.merge(mfcc_df, on='file_path', how='left')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_path</th>\n",
       "      <th>category</th>\n",
       "      <th>mfcc_0</th>\n",
       "      <th>mfcc_1</th>\n",
       "      <th>mfcc_2</th>\n",
       "      <th>mfcc_3</th>\n",
       "      <th>mfcc_4</th>\n",
       "      <th>mfcc_5</th>\n",
       "      <th>mfcc_6</th>\n",
       "      <th>mfcc_7</th>\n",
       "      <th>mfcc_8</th>\n",
       "      <th>mfcc_9</th>\n",
       "      <th>mfcc_10</th>\n",
       "      <th>mfcc_11</th>\n",
       "      <th>mfcc_12</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>data\\backward\\0165e0e8_nohash_0.wav</td>\n",
       "      <td>backward</td>\n",
       "      <td>-543.961487</td>\n",
       "      <td>29.465919</td>\n",
       "      <td>-0.224312</td>\n",
       "      <td>-2.614496</td>\n",
       "      <td>1.775066</td>\n",
       "      <td>5.812069</td>\n",
       "      <td>-0.475146</td>\n",
       "      <td>1.754618</td>\n",
       "      <td>1.698112</td>\n",
       "      <td>1.871549</td>\n",
       "      <td>0.780310</td>\n",
       "      <td>0.555312</td>\n",
       "      <td>0.740507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>data\\backward\\017c4098_nohash_0.wav</td>\n",
       "      <td>backward</td>\n",
       "      <td>-514.944763</td>\n",
       "      <td>22.943722</td>\n",
       "      <td>-2.943335</td>\n",
       "      <td>0.380538</td>\n",
       "      <td>-1.639417</td>\n",
       "      <td>-0.303708</td>\n",
       "      <td>-3.245466</td>\n",
       "      <td>-1.001900</td>\n",
       "      <td>0.160957</td>\n",
       "      <td>-0.656518</td>\n",
       "      <td>-0.744881</td>\n",
       "      <td>-1.379814</td>\n",
       "      <td>0.494081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>data\\backward\\017c4098_nohash_1.wav</td>\n",
       "      <td>backward</td>\n",
       "      <td>-564.731384</td>\n",
       "      <td>24.316267</td>\n",
       "      <td>-4.805779</td>\n",
       "      <td>1.355843</td>\n",
       "      <td>-3.137849</td>\n",
       "      <td>-0.731808</td>\n",
       "      <td>-3.902590</td>\n",
       "      <td>-1.036058</td>\n",
       "      <td>0.867936</td>\n",
       "      <td>-1.381379</td>\n",
       "      <td>-0.920068</td>\n",
       "      <td>-0.432483</td>\n",
       "      <td>0.251636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>data\\backward\\017c4098_nohash_2.wav</td>\n",
       "      <td>backward</td>\n",
       "      <td>-519.679565</td>\n",
       "      <td>24.080870</td>\n",
       "      <td>-3.222754</td>\n",
       "      <td>1.211598</td>\n",
       "      <td>-0.023732</td>\n",
       "      <td>-0.661048</td>\n",
       "      <td>-3.239309</td>\n",
       "      <td>-1.242702</td>\n",
       "      <td>-0.020317</td>\n",
       "      <td>-0.827836</td>\n",
       "      <td>-0.380631</td>\n",
       "      <td>-0.757278</td>\n",
       "      <td>0.135732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>data\\backward\\017c4098_nohash_3.wav</td>\n",
       "      <td>backward</td>\n",
       "      <td>-523.218933</td>\n",
       "      <td>20.803392</td>\n",
       "      <td>-5.167766</td>\n",
       "      <td>-0.932550</td>\n",
       "      <td>-2.657047</td>\n",
       "      <td>0.153993</td>\n",
       "      <td>-4.686269</td>\n",
       "      <td>-1.573421</td>\n",
       "      <td>0.026947</td>\n",
       "      <td>-2.086370</td>\n",
       "      <td>-1.115192</td>\n",
       "      <td>0.045962</td>\n",
       "      <td>-0.660579</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             file_path  category      mfcc_0     mfcc_1  \\\n",
       "0  data\\backward\\0165e0e8_nohash_0.wav  backward -543.961487  29.465919   \n",
       "1  data\\backward\\017c4098_nohash_0.wav  backward -514.944763  22.943722   \n",
       "2  data\\backward\\017c4098_nohash_1.wav  backward -564.731384  24.316267   \n",
       "3  data\\backward\\017c4098_nohash_2.wav  backward -519.679565  24.080870   \n",
       "4  data\\backward\\017c4098_nohash_3.wav  backward -523.218933  20.803392   \n",
       "\n",
       "     mfcc_2    mfcc_3    mfcc_4    mfcc_5    mfcc_6    mfcc_7    mfcc_8  \\\n",
       "0 -0.224312 -2.614496  1.775066  5.812069 -0.475146  1.754618  1.698112   \n",
       "1 -2.943335  0.380538 -1.639417 -0.303708 -3.245466 -1.001900  0.160957   \n",
       "2 -4.805779  1.355843 -3.137849 -0.731808 -3.902590 -1.036058  0.867936   \n",
       "3 -3.222754  1.211598 -0.023732 -0.661048 -3.239309 -1.242702 -0.020317   \n",
       "4 -5.167766 -0.932550 -2.657047  0.153993 -4.686269 -1.573421  0.026947   \n",
       "\n",
       "     mfcc_9   mfcc_10   mfcc_11   mfcc_12  \n",
       "0  1.871549  0.780310  0.555312  0.740507  \n",
       "1 -0.656518 -0.744881 -1.379814  0.494081  \n",
       "2 -1.381379 -0.920068 -0.432483  0.251636  \n",
       "3 -0.827836 -0.380631 -0.757278  0.135732  \n",
       "4 -2.086370 -1.115192  0.045962 -0.660579  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[1], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m# Save as CSV\u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m \u001b[43mdf\u001b[49m\u001b[38;5;241m.\u001b[39mto_csv(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mprocessed_data.csv\u001b[39m\u001b[38;5;124m'\u001b[39m, index\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'df' is not defined"
     ]
    }
   ],
   "source": [
    "# Save as CSV\n",
    "df.to_csv('processed_data.csv', index=False)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('processed_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_path</th>\n",
       "      <th>category</th>\n",
       "      <th>mfcc_0</th>\n",
       "      <th>mfcc_1</th>\n",
       "      <th>mfcc_2</th>\n",
       "      <th>mfcc_3</th>\n",
       "      <th>mfcc_4</th>\n",
       "      <th>mfcc_5</th>\n",
       "      <th>mfcc_6</th>\n",
       "      <th>mfcc_7</th>\n",
       "      <th>mfcc_8</th>\n",
       "      <th>mfcc_9</th>\n",
       "      <th>mfcc_10</th>\n",
       "      <th>mfcc_11</th>\n",
       "      <th>mfcc_12</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>data\\backward\\0165e0e8_nohash_0.wav</td>\n",
       "      <td>backward</td>\n",
       "      <td>-543.96150</td>\n",
       "      <td>29.465920</td>\n",
       "      <td>-0.224312</td>\n",
       "      <td>-2.614497</td>\n",
       "      <td>1.775066</td>\n",
       "      <td>5.812069</td>\n",
       "      <td>-0.475146</td>\n",
       "      <td>1.754618</td>\n",
       "      <td>1.698111</td>\n",
       "      <td>1.871549</td>\n",
       "      <td>0.780310</td>\n",
       "      <td>0.555312</td>\n",
       "      <td>0.740507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>data\\backward\\017c4098_nohash_0.wav</td>\n",
       "      <td>backward</td>\n",
       "      <td>-514.94476</td>\n",
       "      <td>22.943722</td>\n",
       "      <td>-2.943335</td>\n",
       "      <td>0.380538</td>\n",
       "      <td>-1.639417</td>\n",
       "      <td>-0.303708</td>\n",
       "      <td>-3.245466</td>\n",
       "      <td>-1.001900</td>\n",
       "      <td>0.160957</td>\n",
       "      <td>-0.656518</td>\n",
       "      <td>-0.744881</td>\n",
       "      <td>-1.379814</td>\n",
       "      <td>0.494081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>data\\backward\\017c4098_nohash_1.wav</td>\n",
       "      <td>backward</td>\n",
       "      <td>-564.73140</td>\n",
       "      <td>24.316267</td>\n",
       "      <td>-4.805779</td>\n",
       "      <td>1.355843</td>\n",
       "      <td>-3.137849</td>\n",
       "      <td>-0.731808</td>\n",
       "      <td>-3.902590</td>\n",
       "      <td>-1.036058</td>\n",
       "      <td>0.867936</td>\n",
       "      <td>-1.381379</td>\n",
       "      <td>-0.920068</td>\n",
       "      <td>-0.432484</td>\n",
       "      <td>0.251636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>data\\backward\\017c4098_nohash_2.wav</td>\n",
       "      <td>backward</td>\n",
       "      <td>-519.67957</td>\n",
       "      <td>24.080870</td>\n",
       "      <td>-3.222754</td>\n",
       "      <td>1.211598</td>\n",
       "      <td>-0.023732</td>\n",
       "      <td>-0.661048</td>\n",
       "      <td>-3.239309</td>\n",
       "      <td>-1.242702</td>\n",
       "      <td>-0.020317</td>\n",
       "      <td>-0.827836</td>\n",
       "      <td>-0.380631</td>\n",
       "      <td>-0.757278</td>\n",
       "      <td>0.135732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>data\\backward\\017c4098_nohash_3.wav</td>\n",
       "      <td>backward</td>\n",
       "      <td>-523.21893</td>\n",
       "      <td>20.803392</td>\n",
       "      <td>-5.167766</td>\n",
       "      <td>-0.932550</td>\n",
       "      <td>-2.657047</td>\n",
       "      <td>0.153993</td>\n",
       "      <td>-4.686269</td>\n",
       "      <td>-1.573421</td>\n",
       "      <td>0.026947</td>\n",
       "      <td>-2.086371</td>\n",
       "      <td>-1.115192</td>\n",
       "      <td>0.045962</td>\n",
       "      <td>-0.660579</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             file_path  category     mfcc_0     mfcc_1  \\\n",
       "0  data\\backward\\0165e0e8_nohash_0.wav  backward -543.96150  29.465920   \n",
       "1  data\\backward\\017c4098_nohash_0.wav  backward -514.94476  22.943722   \n",
       "2  data\\backward\\017c4098_nohash_1.wav  backward -564.73140  24.316267   \n",
       "3  data\\backward\\017c4098_nohash_2.wav  backward -519.67957  24.080870   \n",
       "4  data\\backward\\017c4098_nohash_3.wav  backward -523.21893  20.803392   \n",
       "\n",
       "     mfcc_2    mfcc_3    mfcc_4    mfcc_5    mfcc_6    mfcc_7    mfcc_8  \\\n",
       "0 -0.224312 -2.614497  1.775066  5.812069 -0.475146  1.754618  1.698111   \n",
       "1 -2.943335  0.380538 -1.639417 -0.303708 -3.245466 -1.001900  0.160957   \n",
       "2 -4.805779  1.355843 -3.137849 -0.731808 -3.902590 -1.036058  0.867936   \n",
       "3 -3.222754  1.211598 -0.023732 -0.661048 -3.239309 -1.242702 -0.020317   \n",
       "4 -5.167766 -0.932550 -2.657047  0.153993 -4.686269 -1.573421  0.026947   \n",
       "\n",
       "     mfcc_9   mfcc_10   mfcc_11   mfcc_12  \n",
       "0  1.871549  0.780310  0.555312  0.740507  \n",
       "1 -0.656518 -0.744881 -1.379814  0.494081  \n",
       "2 -1.381379 -0.920068 -0.432484  0.251636  \n",
       "3 -0.827836 -0.380631 -0.757278  0.135732  \n",
       "4 -2.086371 -1.115192  0.045962 -0.660579  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(105835, 15)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Conv1D, Dropout, Flatten,BatchNormalization\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data preparation complete. Ready for training.\n"
     ]
    }
   ],
   "source": [
    "# Assume df has the columns: file_path, category, mfcc_0 to mfcc_12\n",
    "\n",
    "# Extract MFCC features and labels\n",
    "X = df.iloc[:, 2:].values  # MFCC features (columns mfcc_0 to mfcc_12)\n",
    "y = df['category'].values  # Labels (category)\n",
    "\n",
    "# Encode the labels as integers\n",
    "label_encoder = LabelEncoder()\n",
    "y = label_encoder.fit_transform(y)\n",
    "\n",
    "# Convert labels to one-hot encoding\n",
    "NUM_CLASSES = 36  # Assuming there are 36 unique categories\n",
    "y_one_hot = to_categorical(y, num_classes=NUM_CLASSES)\n",
    "\n",
    "# Split the data into training and testing sets (80% training, 20% testing)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y_one_hot, test_size=0.2, random_state=42)\n",
    "\n",
    "# Reshape X to be compatible with Conv1D layers (samples, timesteps, features)\n",
    "X_train = np.expand_dims(X_train, axis=-1)\n",
    "X_test = np.expand_dims(X_test, axis=-1)\n",
    "\n",
    "print(\"Data preparation complete. Ready for training.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a more complex TDNN model\n",
    "def build_improved_tdnn(input_shape, num_classes):\n",
    "    model = Sequential()\n",
    "\n",
    "    # First TDNN Layer (Conv1D)\n",
    "    model.add(Conv1D(filters=128, kernel_size=5, strides=1, padding='same', activation='relu', input_shape=input_shape))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Dropout(0.3))\n",
    "\n",
    "    # Second TDNN Layer (Conv1D)\n",
    "    model.add(Conv1D(filters=256, kernel_size=3, strides=1, padding='same', activation='relu'))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Dropout(0.3))\n",
    "\n",
    "    # Third TDNN Layer (Conv1D)\n",
    "    model.add(Conv1D(filters=256, kernel_size=3, strides=1, padding='same', activation='relu'))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Dropout(0.3))\n",
    "\n",
    "    # Flatten for fully connected layers\n",
    "    model.add(Flatten())\n",
    "\n",
    "    # First fully connected layer\n",
    "    model.add(Dense(512, activation='relu'))\n",
    "    model.add(Dropout(0.4))\n",
    "\n",
    "    # Second fully connected layer\n",
    "    model.add(Dense(256, activation='relu'))\n",
    "    model.add(Dropout(0.4))\n",
    "\n",
    "    # Output layer for classification\n",
    "    model.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "662/662 - 38s - loss: 1.9871 - accuracy: 0.3387 - val_loss: 1.0158 - val_accuracy: 0.6090 - 38s/epoch - 58ms/step\n",
      "Epoch 2/50\n",
      "662/662 - 37s - loss: 1.1863 - accuracy: 0.5514 - val_loss: 0.8801 - val_accuracy: 0.6313 - 37s/epoch - 56ms/step\n",
      "Epoch 3/50\n",
      "662/662 - 37s - loss: 1.0076 - accuracy: 0.6131 - val_loss: 0.6784 - val_accuracy: 0.7296 - 37s/epoch - 56ms/step\n",
      "Epoch 4/50\n",
      "662/662 - 37s - loss: 0.9151 - accuracy: 0.6481 - val_loss: 0.6090 - val_accuracy: 0.7767 - 37s/epoch - 56ms/step\n",
      "Epoch 5/50\n",
      "662/662 - 37s - loss: 0.8675 - accuracy: 0.6669 - val_loss: 0.7059 - val_accuracy: 0.7011 - 37s/epoch - 56ms/step\n",
      "Epoch 6/50\n",
      "662/662 - 38s - loss: 0.7913 - accuracy: 0.6964 - val_loss: 0.4383 - val_accuracy: 0.8472 - 38s/epoch - 57ms/step\n",
      "Epoch 7/50\n",
      "662/662 - 40s - loss: 0.7619 - accuracy: 0.7093 - val_loss: 0.3568 - val_accuracy: 0.9024 - 40s/epoch - 61ms/step\n",
      "Epoch 8/50\n",
      "662/662 - 41s - loss: 0.7052 - accuracy: 0.7313 - val_loss: 0.3772 - val_accuracy: 0.8882 - 41s/epoch - 61ms/step\n",
      "Epoch 9/50\n",
      "662/662 - 39s - loss: 0.6816 - accuracy: 0.7406 - val_loss: 0.4030 - val_accuracy: 0.8636 - 39s/epoch - 59ms/step\n",
      "Epoch 10/50\n",
      "662/662 - 40s - loss: 0.6418 - accuracy: 0.7559 - val_loss: 0.3637 - val_accuracy: 0.8738 - 40s/epoch - 61ms/step\n",
      "Epoch 11/50\n",
      "662/662 - 40s - loss: 0.6151 - accuracy: 0.7684 - val_loss: 0.2532 - val_accuracy: 0.9413 - 40s/epoch - 61ms/step\n",
      "Epoch 12/50\n",
      "662/662 - 41s - loss: 0.5860 - accuracy: 0.7788 - val_loss: 0.2428 - val_accuracy: 0.9290 - 41s/epoch - 62ms/step\n",
      "Epoch 13/50\n",
      "662/662 - 40s - loss: 0.5781 - accuracy: 0.7826 - val_loss: 0.2807 - val_accuracy: 0.9157 - 40s/epoch - 61ms/step\n",
      "Epoch 14/50\n",
      "662/662 - 40s - loss: 0.5574 - accuracy: 0.7920 - val_loss: 0.3123 - val_accuracy: 0.8925 - 40s/epoch - 60ms/step\n",
      "Epoch 15/50\n",
      "662/662 - 40s - loss: 0.5367 - accuracy: 0.7998 - val_loss: 0.2058 - val_accuracy: 0.9330 - 40s/epoch - 60ms/step\n",
      "Epoch 16/50\n",
      "662/662 - 41s - loss: 0.5197 - accuracy: 0.8075 - val_loss: 0.1773 - val_accuracy: 0.9501 - 41s/epoch - 62ms/step\n",
      "Epoch 17/50\n",
      "662/662 - 41s - loss: 0.4844 - accuracy: 0.8231 - val_loss: 0.1689 - val_accuracy: 0.9575 - 41s/epoch - 62ms/step\n",
      "Epoch 18/50\n",
      "662/662 - 39s - loss: 0.4909 - accuracy: 0.8194 - val_loss: 0.2716 - val_accuracy: 0.9253 - 39s/epoch - 60ms/step\n",
      "Epoch 19/50\n",
      "662/662 - 39s - loss: 0.4746 - accuracy: 0.8262 - val_loss: 0.1909 - val_accuracy: 0.9331 - 39s/epoch - 59ms/step\n",
      "Epoch 20/50\n",
      "662/662 - 39s - loss: 0.4617 - accuracy: 0.8328 - val_loss: 0.2852 - val_accuracy: 0.8937 - 39s/epoch - 59ms/step\n",
      "Epoch 21/50\n",
      "662/662 - 39s - loss: 0.4438 - accuracy: 0.8393 - val_loss: 0.1574 - val_accuracy: 0.9531 - 39s/epoch - 59ms/step\n",
      "Epoch 22/50\n",
      "662/662 - 39s - loss: 0.4413 - accuracy: 0.8394 - val_loss: 0.1648 - val_accuracy: 0.9566 - 39s/epoch - 60ms/step\n",
      "Epoch 23/50\n",
      "662/662 - 39s - loss: 0.4180 - accuracy: 0.8483 - val_loss: 0.1402 - val_accuracy: 0.9690 - 39s/epoch - 59ms/step\n",
      "Epoch 24/50\n",
      "662/662 - 39s - loss: 0.4200 - accuracy: 0.8480 - val_loss: 0.1557 - val_accuracy: 0.9546 - 39s/epoch - 60ms/step\n",
      "Epoch 25/50\n",
      "662/662 - 39s - loss: 0.4030 - accuracy: 0.8541 - val_loss: 0.1304 - val_accuracy: 0.9716 - 39s/epoch - 59ms/step\n",
      "Epoch 26/50\n",
      "662/662 - 39s - loss: 0.3920 - accuracy: 0.8587 - val_loss: 0.1131 - val_accuracy: 0.9749 - 39s/epoch - 60ms/step\n",
      "Epoch 27/50\n",
      "662/662 - 39s - loss: 0.3824 - accuracy: 0.8637 - val_loss: 0.1221 - val_accuracy: 0.9711 - 39s/epoch - 59ms/step\n",
      "Epoch 28/50\n",
      "662/662 - 39s - loss: 0.3857 - accuracy: 0.8615 - val_loss: 0.1267 - val_accuracy: 0.9731 - 39s/epoch - 59ms/step\n",
      "Epoch 29/50\n",
      "662/662 - 44s - loss: 0.3885 - accuracy: 0.8603 - val_loss: 0.1101 - val_accuracy: 0.9679 - 44s/epoch - 66ms/step\n",
      "Epoch 30/50\n",
      "662/662 - 43s - loss: 0.3670 - accuracy: 0.8692 - val_loss: 0.1263 - val_accuracy: 0.9613 - 43s/epoch - 65ms/step\n",
      "Epoch 31/50\n",
      "662/662 - 42s - loss: 0.3451 - accuracy: 0.8769 - val_loss: 0.1070 - val_accuracy: 0.9743 - 42s/epoch - 63ms/step\n",
      "Epoch 32/50\n",
      "662/662 - 42s - loss: 0.3483 - accuracy: 0.8763 - val_loss: 0.0978 - val_accuracy: 0.9735 - 42s/epoch - 63ms/step\n",
      "Epoch 33/50\n",
      "662/662 - 44s - loss: 0.3407 - accuracy: 0.8784 - val_loss: 0.1532 - val_accuracy: 0.9462 - 44s/epoch - 67ms/step\n",
      "Epoch 34/50\n",
      "662/662 - 52s - loss: 0.3584 - accuracy: 0.8725 - val_loss: 0.0902 - val_accuracy: 0.9764 - 52s/epoch - 79ms/step\n",
      "Epoch 35/50\n",
      "662/662 - 48s - loss: 0.3285 - accuracy: 0.8831 - val_loss: 0.0963 - val_accuracy: 0.9741 - 48s/epoch - 72ms/step\n",
      "Epoch 36/50\n",
      "662/662 - 46s - loss: 0.3258 - accuracy: 0.8844 - val_loss: 0.0993 - val_accuracy: 0.9716 - 46s/epoch - 70ms/step\n",
      "Epoch 37/50\n",
      "662/662 - 46s - loss: 0.3282 - accuracy: 0.8837 - val_loss: 0.1175 - val_accuracy: 0.9669 - 46s/epoch - 69ms/step\n",
      "Epoch 38/50\n",
      "662/662 - 49s - loss: 0.3205 - accuracy: 0.8865 - val_loss: 0.1001 - val_accuracy: 0.9697 - 49s/epoch - 73ms/step\n",
      "Epoch 39/50\n",
      "662/662 - 44s - loss: 0.3296 - accuracy: 0.8850 - val_loss: 0.0853 - val_accuracy: 0.9790 - 44s/epoch - 67ms/step\n",
      "Epoch 40/50\n",
      "662/662 - 42s - loss: 0.3206 - accuracy: 0.8862 - val_loss: 0.0862 - val_accuracy: 0.9758 - 42s/epoch - 64ms/step\n",
      "Epoch 41/50\n",
      "662/662 - 45s - loss: 0.3142 - accuracy: 0.8895 - val_loss: 0.0868 - val_accuracy: 0.9768 - 45s/epoch - 68ms/step\n",
      "Epoch 42/50\n",
      "662/662 - 45s - loss: 0.3005 - accuracy: 0.8948 - val_loss: 0.1282 - val_accuracy: 0.9551 - 45s/epoch - 68ms/step\n",
      "Epoch 43/50\n",
      "662/662 - 45s - loss: 0.3001 - accuracy: 0.8935 - val_loss: 0.1156 - val_accuracy: 0.9599 - 45s/epoch - 69ms/step\n",
      "Epoch 44/50\n",
      "662/662 - 44s - loss: 0.2867 - accuracy: 0.8988 - val_loss: 0.1191 - val_accuracy: 0.9544 - 44s/epoch - 67ms/step\n"
     ]
    }
   ],
   "source": [
    "# Input shape (for each MFCC feature)\n",
    "input_shape = (X_train.shape[1], 1)\n",
    "\n",
    "# Build the model\n",
    "model = build_improved_tdnn(input_shape, NUM_CLASSES)\n",
    "\n",
    "# Compile the model\n",
    "model.compile(optimizer=Adam(learning_rate=0.001),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "# Add early stopping to avoid overfitting\n",
    "early_stopping = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)\n",
    "\n",
    "# Train the model with a higher number of epochs and more verbose output\n",
    "history = model.fit(X_train, y_train, \n",
    "                    epochs=50,  # Increase the number of epochs\n",
    "                    batch_size=128,  # Adjust based on available memory\n",
    "                    validation_data=(X_test, y_test),\n",
    "                    callbacks=[early_stopping],\n",
    "                    verbose=2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "662/662 - 7s - loss: 0.0853 - accuracy: 0.9790 - 7s/epoch - 11ms/step\n",
      "Test Accuracy: 97.90%\n"
     ]
    }
   ],
   "source": [
    "# Evaluate the model on the test set\n",
    "test_loss, test_accuracy = model.evaluate(X_test, y_test, verbose=2)\n",
    "print(f\"Test Accuracy: {test_accuracy * 100:.2f}%\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: h5py in c:\\users\\hp\\anaconda3\\envs\\py310\\lib\\site-packages (3.11.0)\n",
      "Requirement already satisfied: numpy>=1.17.3 in c:\\users\\hp\\anaconda3\\envs\\py310\\lib\\site-packages (from h5py) (1.23.5)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install h5py\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:Found untraced functions such as _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op while saving (showing 3 of 3). These functions will not be directly callable after loading.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: model\\assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: model\\assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model saved to 'complex_tdnn_model'.\n"
     ]
    }
   ],
   "source": [
    "# Save the trained model in TensorFlow SavedModel format\n",
    "model.save('model')\n",
    "print(\"Model saved to 'complex_tdnn_model'.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py310",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}