scripts/kerala/parse_kerala_eng.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pdf2image\n",
    "from PIL import Image\n",
    "import pytesseract\n",
    "import difflib\n",
    "import re\n",
    "import pandas as pd\n",
    "import sys\n",
    "sys.path.append('../')\n",
    "from helper import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = '../../data/'\n",
    "STATE = 'kerala/'\n",
    "\n",
    "PARSE_DATA_PAGES = \"../../parseData/images/\"+STATE\n",
    "create_path(PARSE_DATA_PAGES)\n",
    "\n",
    "PARSE_DATA_BLOCKS = \"../../parseData/blocks/\"+STATE\n",
    "create_path(PARSE_DATA_BLOCKS)\n",
    "\n",
    "PARSE_DATA_CSVS = \"../../parseData/csvs/\"+STATE\n",
    "create_path(PARSE_DATA_CSVS)\n",
    "\n",
    "COLUMNS = [\"number\",\"id\", \"elector_name\", \"father_or_husband_name\", \"relationship\", \"house_no\", \"age\", \"sex\", \"ac_name\", \"parl_constituency\", \"part_no\", \"year\", \"state\", \"filename\", \"main_town\", \"police_station\", \"mandal\", \"revenue_division\", \"district\", \"pin_code\", \"polling_station_name\", \"polling_station_address\", \"net_electors_male\", \"net_electors_female\", \"net_electors_third_gender\", \"net_electors_total\"]\n",
    "\n",
    "state_pdfs_path = DATA_PATH+STATE\n",
    "state_pdfs_files = os.listdir(state_pdfs_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['.DS_Store',\n",
       " 'eng_01_001_011.pdf',\n",
       " 'eng_01_001_015.pdf',\n",
       " 'eng_01_001_018.pdf',\n",
       " 'eng_01_001_024.pdf',\n",
       " 'eng_01_001_052.pdf',\n",
       " 'eng_01_001_085.pdf',\n",
       " 'eng_01_001_093.pdf',\n",
       " 'eng_01_001_112.pdf',\n",
       " 'eng_01_001_154.pdf',\n",
       " 'eng_01_001_170.pdf',\n",
       " 'eng_01_001_184.pdf',\n",
       " 'eng_01_002_003.pdf',\n",
       " 'eng_01_002_007.pdf',\n",
       " 'eng_01_002_031.pdf',\n",
       " 'eng_01_002_032.pdf',\n",
       " 'eng_01_002_063.pdf',\n",
       " 'eng_01_002_079.pdf',\n",
       " 'eng_01_002_114.pdf',\n",
       " 'eng_01_002_120.pdf',\n",
       " 'eng_01_002_133.pdf',\n",
       " 'eng_01_002_181.pdf',\n",
       " 'eng_01_003_081.pdf',\n",
       " 'eng_01_003_128.pdf',\n",
       " 'eng_01_003_160.pdf',\n",
       " 'eng_01_003_180.pdf',\n",
       " 'eng_01_004_006.pdf',\n",
       " 'eng_01_004_050.pdf',\n",
       " 'eng_01_004_065.pdf',\n",
       " 'eng_01_004_089.pdf',\n",
       " 'eng_01_004_106.pdf',\n",
       " 'eng_01_004_117.pdf',\n",
       " 'eng_01_004_144.pdf',\n",
       " 'eng_01_005_001.pdf',\n",
       " 'eng_01_005_098.pdf',\n",
       " 'eng_01_005_112.pdf',\n",
       " 'eng_01_005_152.pdf',\n",
       " 'eng_01_005_169.pdf',\n",
       " 'eng_02_006_019.pdf',\n",
       " 'eng_02_006_024.pdf',\n",
       " 'eng_02_006_061.pdf',\n",
       " 'eng_02_006_066.pdf',\n",
       " 'eng_02_006_081.pdf',\n",
       " 'eng_02_006_084.pdf',\n",
       " 'eng_02_006_117.pdf',\n",
       " 'eng_02_006_151.pdf',\n",
       " 'eng_02_006_160.pdf',\n",
       " 'eng_02_006_168.pdf',\n",
       " 'eng_02_007_006.pdf',\n",
       " 'eng_02_007_024.pdf',\n",
       " 'eng_02_007_026.pdf',\n",
       " 'eng_02_007_032.pdf',\n",
       " 'eng_02_007_037.pdf',\n",
       " 'eng_02_007_062.pdf',\n",
       " 'eng_02_007_080.pdf',\n",
       " 'eng_02_007_094.pdf',\n",
       " 'eng_02_007_110.pdf',\n",
       " 'eng_02_007_153.pdf',\n",
       " 'eng_02_008_056.pdf',\n",
       " 'eng_02_008_061.pdf',\n",
       " 'eng_02_008_066.pdf',\n",
       " 'eng_02_008_093.pdf',\n",
       " 'eng_02_008_104.pdf',\n",
       " 'eng_02_008_149.pdf',\n",
       " 'eng_02_008_153.pdf',\n",
       " 'eng_02_009_074.pdf',\n",
       " 'eng_02_009_125.pdf',\n",
       " 'eng_02_010_019.pdf',\n",
       " 'eng_02_010_025.pdf',\n",
       " 'eng_02_010_126.pdf',\n",
       " 'eng_02_010_132.pdf',\n",
       " 'eng_02_010_144.pdf',\n",
       " 'eng_02_011_004.pdf',\n",
       " 'eng_02_011_005.pdf',\n",
       " 'eng_02_011_016.pdf',\n",
       " 'eng_02_011_025.pdf',\n",
       " 'eng_02_011_059.pdf',\n",
       " 'eng_02_011_084.pdf',\n",
       " 'eng_02_011_113.pdf',\n",
       " 'eng_02_011_126.pdf',\n",
       " 'eng_02_011_132.pdf',\n",
       " 'eng_02_011_138.pdf',\n",
       " 'eng_02_012_067.pdf',\n",
       " 'eng_02_012_125.pdf',\n",
       " 'eng_02_012_144.pdf',\n",
       " 'eng_02_013_031.pdf',\n",
       " 'eng_02_013_038.pdf',\n",
       " 'eng_02_013_039.pdf',\n",
       " 'eng_02_013_081.pdf',\n",
       " 'eng_02_013_091.pdf',\n",
       " 'eng_02_013_117.pdf',\n",
       " 'eng_02_013_141.pdf',\n",
       " 'eng_02_014_014.pdf',\n",
       " 'eng_02_014_019.pdf',\n",
       " 'eng_02_014_029.pdf',\n",
       " 'eng_02_014_074.pdf',\n",
       " 'eng_02_014_084.pdf',\n",
       " 'eng_02_014_112.pdf',\n",
       " 'eng_02_014_151.pdf',\n",
       " 'eng_02_014_159.pdf',\n",
       " 'eng_02_014_163.pdf',\n",
       " 'eng_02_015_014.pdf',\n",
       " 'eng_02_015_048.pdf',\n",
       " 'eng_02_015_056.pdf',\n",
       " 'eng_02_015_066.pdf',\n",
       " 'eng_02_015_119.pdf',\n",
       " 'eng_02_015_120.pdf',\n",
       " 'eng_02_015_125.pdf',\n",
       " 'eng_02_016_015.pdf',\n",
       " 'eng_02_016_080.pdf',\n",
       " 'eng_02_016_096.pdf',\n",
       " 'eng_02_016_105.pdf',\n",
       " 'eng_02_016_119.pdf',\n",
       " 'eng_02_016_125.pdf',\n",
       " 'eng_03_017_009.pdf',\n",
       " 'eng_03_017_028.pdf',\n",
       " 'eng_03_017_083.pdf',\n",
       " 'eng_03_017_097.pdf',\n",
       " 'eng_03_017_119.pdf',\n",
       " 'eng_03_017_121.pdf',\n",
       " 'eng_03_017_122.pdf',\n",
       " 'eng_03_017_126.pdf',\n",
       " 'eng_03_017_140.pdf',\n",
       " 'eng_03_017_155.pdf',\n",
       " 'eng_03_018_005.pdf',\n",
       " 'eng_03_018_020.pdf',\n",
       " 'eng_03_018_023.pdf',\n",
       " 'eng_03_018_070.pdf',\n",
       " 'eng_03_018_103.pdf',\n",
       " 'eng_03_018_125.pdf',\n",
       " 'eng_03_018_158.pdf',\n",
       " 'eng_03_018_196.pdf',\n",
       " 'eng_03_019_013.pdf',\n",
       " 'eng_03_019_046.pdf',\n",
       " 'eng_03_019_047.pdf',\n",
       " 'eng_03_019_048.pdf',\n",
       " 'eng_03_019_060.pdf',\n",
       " 'eng_03_019_090.pdf',\n",
       " 'eng_03_019_094.pdf',\n",
       " 'eng_03_019_112.pdf',\n",
       " 'eng_03_019_163.pdf',\n",
       " 'eng_04_020_007.pdf',\n",
       " 'eng_04_020_014.pdf',\n",
       " 'eng_04_020_020.pdf',\n",
       " 'eng_04_020_074.pdf',\n",
       " 'eng_04_020_078.pdf',\n",
       " 'eng_04_020_094.pdf',\n",
       " 'eng_04_020_105.pdf',\n",
       " 'eng_04_020_122.pdf',\n",
       " 'eng_04_021_005.pdf',\n",
       " 'eng_04_021_061.pdf',\n",
       " 'eng_04_021_074.pdf',\n",
       " 'eng_04_021_087.pdf',\n",
       " 'eng_04_021_092.pdf',\n",
       " 'eng_04_021_105.pdf',\n",
       " 'eng_04_021_144.pdf',\n",
       " 'eng_04_022_017.pdf',\n",
       " 'eng_04_022_025.pdf',\n",
       " 'eng_04_022_052.pdf',\n",
       " 'eng_04_022_059.pdf',\n",
       " 'eng_04_022_122.pdf',\n",
       " 'eng_04_022_144.pdf',\n",
       " 'eng_04_022_151.pdf',\n",
       " 'eng_04_022_168.pdf',\n",
       " 'eng_04_022_169.pdf',\n",
       " 'eng_04_022_172.pdf',\n",
       " 'eng_04_022_182.pdf',\n",
       " 'eng_04_023_007.pdf',\n",
       " 'eng_04_023_056.pdf',\n",
       " 'eng_04_023_066.pdf',\n",
       " 'eng_04_023_103.pdf',\n",
       " 'eng_04_023_148.pdf',\n",
       " 'eng_04_023_152.pdf',\n",
       " 'eng_04_024_030.pdf',\n",
       " 'eng_04_024_041.pdf',\n",
       " 'eng_04_024_068.pdf',\n",
       " 'eng_04_024_071.pdf',\n",
       " 'eng_04_024_132.pdf',\n",
       " 'eng_04_024_138.pdf',\n",
       " 'eng_04_024_157.pdf',\n",
       " 'eng_04_025_006.pdf',\n",
       " 'eng_04_025_012.pdf',\n",
       " 'eng_04_025_014.pdf',\n",
       " 'eng_04_025_015.pdf',\n",
       " 'eng_04_025_040.pdf',\n",
       " 'eng_04_025_051.pdf',\n",
       " 'eng_04_025_054.pdf',\n",
       " 'eng_04_025_088.pdf',\n",
       " 'eng_04_025_139.pdf',\n",
       " 'eng_04_025_156.pdf',\n",
       " 'eng_04_026_005.pdf',\n",
       " 'eng_04_026_029.pdf',\n",
       " 'eng_04_026_055.pdf',\n",
       " 'eng_04_026_060.pdf',\n",
       " 'eng_04_026_093.pdf',\n",
       " 'eng_04_026_105.pdf',\n",
       " 'eng_04_026_108.pdf',\n",
       " 'eng_04_026_129.pdf',\n",
       " 'eng_04_027_053.pdf',\n",
       " 'eng_04_027_073.pdf',\n",
       " 'eng_04_027_096.pdf',\n",
       " 'eng_04_027_123.pdf',\n",
       " 'eng_04_027_128.pdf',\n",
       " 'eng_04_027_130.pdf',\n",
       " 'eng_04_027_144.pdf',\n",
       " 'eng_04_028_005.pdf',\n",
       " 'eng_04_028_011.pdf',\n",
       " 'eng_04_028_027.pdf',\n",
       " 'eng_04_028_039.pdf',\n",
       " 'eng_04_028_060.pdf',\n",
       " 'eng_04_028_068.pdf',\n",
       " 'eng_04_028_093.pdf',\n",
       " 'eng_04_028_123.pdf',\n",
       " 'eng_04_029_010.pdf',\n",
       " 'eng_04_029_066.pdf',\n",
       " 'eng_04_029_114.pdf',\n",
       " 'eng_04_030_012.pdf',\n",
       " 'eng_04_030_024.pdf',\n",
       " 'eng_04_030_037.pdf',\n",
       " 'eng_04_030_049.pdf',\n",
       " 'eng_04_030_097.pdf',\n",
       " 'eng_04_030_119.pdf',\n",
       " 'eng_04_030_130.pdf',\n",
       " 'eng_04_030_134.pdf',\n",
       " 'eng_04_030_166.pdf',\n",
       " 'eng_04_030_173.pdf',\n",
       " 'eng_04_030_177.pdf',\n",
       " 'eng_04_031_013.pdf',\n",
       " 'eng_04_031_051.pdf',\n",
       " 'eng_04_031_068.pdf',\n",
       " 'eng_04_031_086.pdf',\n",
       " 'eng_04_031_089.pdf',\n",
       " 'eng_04_031_129.pdf',\n",
       " 'eng_04_031_156.pdf',\n",
       " 'eng_04_031_163.pdf',\n",
       " 'eng_04_032_025.pdf',\n",
       " 'eng_04_032_052.pdf',\n",
       " 'eng_04_032_054.pdf',\n",
       " 'eng_04_032_095.pdf',\n",
       " 'eng_04_032_096.pdf',\n",
       " 'eng_04_032_101.pdf',\n",
       " 'eng_04_032_166.pdf',\n",
       " 'eng_05_033_009.pdf',\n",
       " 'eng_05_033_023.pdf',\n",
       " 'eng_05_033_033.pdf',\n",
       " 'eng_05_033_036.pdf',\n",
       " 'eng_05_033_089.pdf',\n",
       " 'eng_05_033_106.pdf',\n",
       " 'eng_05_033_124.pdf',\n",
       " 'eng_05_033_136.pdf',\n",
       " 'eng_05_033_152.pdf',\n",
       " 'eng_05_033_155.pdf',\n",
       " 'eng_05_033_157.pdf',\n",
       " 'eng_05_034_001.pdf',\n",
       " 'eng_05_034_044.pdf',\n",
       " 'eng_05_034_054.pdf',\n",
       " 'eng_05_034_062.pdf',\n",
       " 'eng_05_034_077.pdf',\n",
       " 'eng_05_034_092.pdf',\n",
       " 'eng_05_034_128.pdf',\n",
       " 'eng_05_034_152.pdf',\n",
       " 'eng_05_035_024.pdf',\n",
       " 'eng_05_035_050.pdf',\n",
       " 'eng_05_035_051.pdf',\n",
       " 'eng_05_035_056.pdf',\n",
       " 'eng_05_035_065.pdf',\n",
       " 'eng_05_035_075.pdf',\n",
       " 'eng_05_035_136.pdf',\n",
       " 'eng_05_035_140.pdf',\n",
       " 'eng_05_035_193.pdf',\n",
       " 'eng_05_036_020.pdf',\n",
       " 'eng_05_036_083.pdf',\n",
       " 'eng_05_036_086.pdf',\n",
       " 'eng_05_036_110.pdf',\n",
       " 'eng_05_036_118.pdf',\n",
       " 'eng_05_036_152.pdf',\n",
       " 'eng_05_036_153.pdf',\n",
       " 'eng_05_036_158.pdf',\n",
       " 'eng_05_036_177.pdf',\n",
       " 'eng_05_036_195.pdf',\n",
       " 'eng_05_037_008.pdf',\n",
       " 'eng_05_037_011.pdf',\n",
       " 'eng_05_037_023.pdf',\n",
       " 'eng_05_037_033.pdf',\n",
       " 'eng_05_037_053.pdf',\n",
       " 'eng_05_037_098.pdf',\n",
       " 'eng_05_037_105.pdf',\n",
       " 'eng_05_037_119.pdf',\n",
       " 'eng_05_037_129.pdf',\n",
       " 'eng_05_037_168.pdf',\n",
       " 'eng_05_038_008.pdf',\n",
       " 'eng_05_038_015.pdf',\n",
       " 'eng_05_038_021.pdf',\n",
       " 'eng_05_038_022.pdf',\n",
       " 'eng_05_038_027.pdf',\n",
       " 'eng_05_038_099.pdf',\n",
       " 'eng_05_038_103.pdf',\n",
       " 'eng_05_038_108.pdf',\n",
       " 'eng_05_038_122.pdf',\n",
       " 'eng_05_038_127.pdf',\n",
       " 'eng_05_038_150.pdf',\n",
       " 'eng_05_039_006.pdf',\n",
       " 'eng_05_039_014.pdf',\n",
       " 'eng_05_039_031.pdf',\n",
       " 'eng_05_039_069.pdf',\n",
       " 'eng_05_039_168.pdf',\n",
       " 'eng_05_039_174.pdf',\n",
       " 'eng_05_040_072.pdf',\n",
       " 'eng_05_040_076.pdf',\n",
       " 'eng_05_040_092.pdf',\n",
       " 'eng_05_040_109.pdf',\n",
       " 'eng_05_040_153.pdf',\n",
       " 'eng_05_041_040.pdf',\n",
       " 'eng_05_041_089.pdf',\n",
       " 'eng_05_041_133.pdf',\n",
       " 'eng_05_042_039.pdf',\n",
       " 'eng_05_042_043.pdf',\n",
       " 'eng_05_042_086.pdf',\n",
       " 'eng_05_042_087.pdf',\n",
       " 'eng_05_042_109.pdf',\n",
       " 'eng_05_042_125.pdf',\n",
       " 'eng_05_042_157.pdf',\n",
       " 'eng_05_043_022.pdf',\n",
       " 'eng_05_043_024.pdf',\n",
       " 'eng_05_043_027.pdf',\n",
       " 'eng_05_043_072.pdf',\n",
       " 'eng_05_043_076.pdf',\n",
       " 'eng_05_043_089.pdf',\n",
       " 'eng_05_043_105.pdf',\n",
       " 'eng_05_043_135.pdf',\n",
       " 'eng_05_044_023.pdf',\n",
       " 'eng_05_044_043.pdf',\n",
       " 'eng_05_044_067.pdf',\n",
       " 'eng_05_044_118.pdf',\n",
       " 'eng_05_045_009.pdf',\n",
       " 'eng_05_045_045.pdf',\n",
       " 'eng_05_045_081.pdf',\n",
       " 'eng_05_045_096.pdf',\n",
       " 'eng_05_045_100.pdf',\n",
       " 'eng_05_045_105.pdf',\n",
       " 'eng_05_045_120.pdf',\n",
       " 'eng_05_045_158.pdf',\n",
       " 'eng_05_045_170.pdf',\n",
       " 'eng_05_046_032.pdf',\n",
       " 'eng_05_046_034.pdf',\n",
       " 'eng_05_046_052.pdf',\n",
       " 'eng_05_046_059.pdf',\n",
       " 'eng_05_046_079.pdf',\n",
       " 'eng_05_046_099.pdf',\n",
       " 'eng_05_046_116.pdf',\n",
       " 'eng_05_046_132.pdf',\n",
       " 'eng_05_046_152.pdf',\n",
       " 'eng_05_047_034.pdf',\n",
       " 'eng_05_047_099.pdf',\n",
       " 'eng_05_048_043.pdf',\n",
       " 'eng_05_048_050.pdf',\n",
       " 'eng_05_048_081.pdf',\n",
       " 'eng_05_048_089.pdf',\n",
       " 'eng_05_048_096.pdf',\n",
       " 'eng_05_048_124.pdf',\n",
       " 'eng_06_049_029.pdf',\n",
       " 'eng_06_049_067.pdf',\n",
       " 'eng_06_049_076.pdf',\n",
       " 'eng_06_049_105.pdf',\n",
       " 'eng_06_049_129.pdf',\n",
       " 'eng_06_049_132.pdf',\n",
       " 'eng_06_049_145.pdf',\n",
       " 'eng_06_049_155.pdf',\n",
       " 'eng_06_050_021.pdf',\n",
       " 'eng_06_050_039.pdf',\n",
       " 'eng_06_050_090.pdf',\n",
       " 'eng_06_050_124.pdf',\n",
       " 'eng_06_050_132.pdf',\n",
       " 'eng_06_050_148.pdf',\n",
       " 'eng_06_051_017.pdf',\n",
       " 'eng_06_051_034.pdf',\n",
       " 'eng_06_051_055.pdf',\n",
       " 'eng_06_051_066.pdf',\n",
       " 'eng_06_051_097.pdf',\n",
       " 'eng_06_051_102.pdf',\n",
       " 'eng_06_051_121.pdf',\n",
       " 'eng_06_051_141.pdf',\n",
       " 'eng_06_051_172.pdf',\n",
       " 'eng_06_052_013.pdf',\n",
       " 'eng_06_052_017.pdf',\n",
       " 'eng_06_052_031.pdf',\n",
       " 'eng_06_052_096.pdf',\n",
       " 'eng_06_052_123.pdf',\n",
       " 'eng_06_052_149.pdf',\n",
       " 'eng_06_052_162.pdf',\n",
       " 'eng_06_052_168.pdf',\n",
       " 'eng_06_052_170.pdf',\n",
       " 'eng_06_052_179.pdf',\n",
       " 'eng_06_052_180.pdf',\n",
       " 'eng_06_052_185.pdf',\n",
       " 'eng_06_053_005.pdf',\n",
       " 'eng_06_053_014.pdf',\n",
       " 'eng_06_053_033.pdf',\n",
       " 'eng_06_053_091.pdf',\n",
       " 'eng_06_053_095.pdf',\n",
       " 'eng_06_053_101.pdf',\n",
       " 'eng_06_053_158.pdf',\n",
       " 'eng_06_054_027.pdf',\n",
       " 'eng_06_054_057.pdf',\n",
       " 'eng_06_054_074.pdf',\n",
       " 'eng_06_054_131.pdf',\n",
       " 'eng_06_054_139.pdf',\n",
       " 'eng_06_054_167.pdf',\n",
       " 'eng_06_055_012.pdf',\n",
       " 'eng_06_055_050.pdf',\n",
       " 'eng_06_055_069.pdf',\n",
       " 'eng_06_055_076.pdf',\n",
       " 'eng_06_055_079.pdf',\n",
       " 'eng_06_055_084.pdf',\n",
       " 'eng_06_055_107.pdf',\n",
       " 'eng_06_055_134.pdf',\n",
       " 'eng_06_055_139.pdf',\n",
       " 'eng_06_055_147.pdf',\n",
       " 'eng_06_055_155.pdf',\n",
       " 'eng_06_055_194.pdf',\n",
       " 'eng_06_055_209.pdf',\n",
       " 'eng_06_056_014.pdf',\n",
       " 'eng_06_056_070.pdf',\n",
       " 'eng_06_056_071.pdf',\n",
       " 'eng_06_056_081.pdf',\n",
       " 'eng_06_056_097.pdf',\n",
       " 'eng_06_056_132.pdf',\n",
       " 'eng_06_056_137.pdf',\n",
       " 'eng_06_056_144.pdf',\n",
       " 'eng_06_056_147.pdf',\n",
       " 'eng_06_056_176.pdf',\n",
       " 'eng_06_057_005.pdf',\n",
       " 'eng_06_057_016.pdf',\n",
       " 'eng_06_057_020.pdf',\n",
       " 'eng_06_057_090.pdf',\n",
       " 'eng_06_057_120.pdf',\n",
       " 'eng_06_057_123.pdf',\n",
       " 'eng_06_058_050.pdf',\n",
       " 'eng_06_058_123.pdf',\n",
       " 'eng_06_058_129.pdf',\n",
       " 'eng_06_058_151.pdf',\n",
       " 'eng_06_058_154.pdf',\n",
       " 'eng_06_059_029.pdf',\n",
       " 'eng_06_059_041.pdf',\n",
       " 'eng_06_059_071.pdf',\n",
       " 'eng_06_059_104.pdf',\n",
       " 'eng_06_059_131.pdf',\n",
       " 'eng_06_059_133.pdf',\n",
       " 'eng_06_059_146.pdf',\n",
       " 'eng_06_059_151.pdf',\n",
       " 'eng_06_059_160.pdf',\n",
       " 'eng_06_059_170.pdf',\n",
       " 'eng_06_060_016.pdf',\n",
       " 'eng_06_060_018.pdf',\n",
       " 'eng_06_060_024.pdf',\n",
       " 'eng_06_060_035.pdf',\n",
       " 'eng_06_060_054.pdf',\n",
       " 'eng_06_060_060.pdf',\n",
       " 'eng_06_060_117.pdf',\n",
       " 'eng_06_060_121.pdf',\n",
       " 'eng_07_061_061.pdf',\n",
       " 'eng_07_061_075.pdf',\n",
       " 'eng_07_061_102.pdf',\n",
       " 'eng_07_061_107.pdf',\n",
       " 'eng_07_061_128.pdf',\n",
       " 'eng_07_061_138.pdf',\n",
       " 'eng_07_061_150.pdf',\n",
       " 'eng_07_061_153.pdf',\n",
       " 'eng_07_061_157.pdf',\n",
       " 'eng_07_062_008.pdf',\n",
       " 'eng_07_062_027.pdf',\n",
       " 'eng_07_062_039.pdf',\n",
       " 'eng_07_062_055.pdf',\n",
       " 'eng_07_062_078.pdf',\n",
       " 'eng_07_062_083.pdf',\n",
       " 'eng_07_062_084.pdf',\n",
       " 'eng_07_062_110.pdf',\n",
       " 'eng_07_062_130.pdf',\n",
       " 'eng_07_063_006.pdf',\n",
       " 'eng_07_063_011.pdf',\n",
       " 'eng_07_063_014.pdf',\n",
       " 'eng_07_063_021.pdf',\n",
       " 'eng_07_063_027.pdf',\n",
       " 'eng_07_063_029.pdf',\n",
       " 'eng_07_063_047.pdf',\n",
       " 'eng_07_063_050.pdf',\n",
       " 'eng_07_063_056.pdf',\n",
       " 'eng_07_063_064.pdf',\n",
       " 'eng_07_063_068.pdf',\n",
       " 'eng_07_063_074.pdf',\n",
       " 'eng_07_063_131.pdf',\n",
       " 'eng_07_063_141.pdf',\n",
       " 'eng_07_063_178.pdf',\n",
       " 'eng_07_064_108.pdf',\n",
       " 'eng_07_064_133.pdf',\n",
       " 'eng_07_064_142.pdf',\n",
       " 'eng_07_064_147.pdf',\n",
       " 'eng_07_064_152.pdf',\n",
       " 'eng_07_064_178.pdf',\n",
       " 'eng_07_064_184.pdf',\n",
       " 'eng_07_065_025.pdf',\n",
       " 'eng_07_065_034.pdf',\n",
       " 'eng_07_065_039.pdf',\n",
       " 'eng_07_065_050.pdf',\n",
       " 'eng_07_065_101.pdf',\n",
       " 'eng_07_065_117.pdf',\n",
       " 'eng_07_065_136.pdf',\n",
       " 'eng_07_065_148.pdf',\n",
       " 'eng_07_065_165.pdf',\n",
       " 'eng_07_066_004.pdf',\n",
       " 'eng_07_066_007.pdf',\n",
       " 'eng_07_066_011.pdf',\n",
       " 'eng_07_066_016.pdf',\n",
       " 'eng_07_066_047.pdf',\n",
       " 'eng_07_066_058.pdf',\n",
       " 'eng_07_066_094.pdf',\n",
       " 'eng_07_066_139.pdf',\n",
       " 'eng_07_067_020.pdf',\n",
       " 'eng_07_067_080.pdf',\n",
       " 'eng_07_067_096.pdf',\n",
       " 'eng_07_067_106.pdf',\n",
       " 'eng_07_067_138.pdf',\n",
       " 'eng_07_067_142.pdf',\n",
       " 'eng_07_068_009.pdf',\n",
       " 'eng_07_068_018.pdf',\n",
       " 'eng_07_068_041.pdf',\n",
       " 'eng_07_068_044.pdf',\n",
       " 'eng_07_068_046.pdf',\n",
       " 'eng_07_068_047.pdf',\n",
       " 'eng_07_068_050.pdf',\n",
       " 'eng_07_068_084.pdf',\n",
       " 'eng_07_068_111.pdf',\n",
       " 'eng_07_068_116.pdf',\n",
       " 'eng_07_068_117.pdf',\n",
       " 'eng_07_068_118.pdf',\n",
       " 'eng_07_068_169.pdf',\n",
       " 'eng_07_069_031.pdf',\n",
       " 'eng_07_069_043.pdf',\n",
       " 'eng_07_069_112.pdf',\n",
       " 'eng_07_069_118.pdf',\n",
       " 'eng_07_069_124.pdf',\n",
       " 'eng_07_070_009.pdf',\n",
       " 'eng_07_070_028.pdf',\n",
       " 'eng_07_070_046.pdf',\n",
       " 'eng_07_070_068.pdf',\n",
       " 'eng_07_070_081.pdf',\n",
       " 'eng_07_070_091.pdf',\n",
       " 'eng_07_070_093.pdf',\n",
       " 'eng_07_070_095.pdf',\n",
       " 'eng_07_070_100.pdf',\n",
       " 'eng_07_070_125.pdf',\n",
       " 'eng_07_070_164.pdf',\n",
       " 'eng_07_070_171.pdf',\n",
       " 'eng_07_071_002.pdf',\n",
       " 'eng_07_071_016.pdf',\n",
       " 'eng_07_071_029.pdf',\n",
       " 'eng_07_071_046.pdf',\n",
       " 'eng_07_071_058.pdf',\n",
       " 'eng_07_071_072.pdf',\n",
       " 'eng_07_071_084.pdf',\n",
       " 'eng_07_071_106.pdf',\n",
       " 'eng_07_071_134.pdf',\n",
       " 'eng_07_071_150.pdf',\n",
       " 'eng_07_071_152.pdf',\n",
       " 'eng_07_071_168.pdf',\n",
       " 'eng_07_072_002.pdf',\n",
       " 'eng_07_072_016.pdf',\n",
       " 'eng_07_072_032.pdf',\n",
       " 'eng_07_072_046.pdf',\n",
       " 'eng_07_072_064.pdf',\n",
       " 'eng_07_072_075.pdf',\n",
       " 'eng_07_072_083.pdf',\n",
       " 'eng_07_072_087.pdf',\n",
       " 'eng_07_072_107.pdf',\n",
       " 'eng_07_072_114.pdf',\n",
       " 'eng_07_072_159.pdf',\n",
       " 'eng_07_073_003.pdf',\n",
       " 'eng_07_073_006.pdf',\n",
       " 'eng_07_073_011.pdf',\n",
       " 'eng_07_073_013.pdf',\n",
       " 'eng_07_073_017.pdf',\n",
       " 'eng_07_073_044.pdf',\n",
       " 'eng_07_073_111.pdf',\n",
       " 'eng_07_073_132.pdf',\n",
       " 'eng_07_073_157.pdf',\n",
       " 'eng_08_074_139.pdf',\n",
       " 'eng_08_074_157.pdf',\n",
       " 'eng_08_075_027.pdf',\n",
       " 'eng_08_075_048.pdf',\n",
       " 'eng_08_075_051.pdf',\n",
       " 'eng_08_075_054.pdf',\n",
       " 'eng_08_075_093.pdf',\n",
       " 'eng_08_075_095.pdf',\n",
       " 'eng_08_075_105.pdf',\n",
       " 'eng_08_076_001.pdf',\n",
       " 'eng_08_076_008.pdf',\n",
       " 'eng_08_076_046.pdf',\n",
       " 'eng_08_076_052.pdf',\n",
       " 'eng_08_076_083.pdf',\n",
       " 'eng_08_076_089.pdf',\n",
       " 'eng_08_076_090.pdf',\n",
       " 'eng_08_076_126.pdf',\n",
       " 'eng_08_076_156.pdf',\n",
       " 'eng_08_076_157.pdf',\n",
       " 'eng_08_077_006.pdf',\n",
       " 'eng_08_077_066.pdf',\n",
       " 'eng_08_077_068.pdf',\n",
       " 'eng_08_077_086.pdf',\n",
       " 'eng_08_077_128.pdf',\n",
       " 'eng_08_077_163.pdf',\n",
       " 'eng_08_078_037.pdf',\n",
       " 'eng_08_078_082.pdf',\n",
       " 'eng_08_078_114.pdf',\n",
       " 'eng_08_078_139.pdf',\n",
       " 'eng_08_078_140.pdf',\n",
       " 'eng_08_078_144.pdf',\n",
       " 'eng_08_079_024.pdf',\n",
       " 'eng_08_079_030.pdf',\n",
       " 'eng_08_079_048.pdf',\n",
       " 'eng_08_079_087.pdf',\n",
       " 'eng_08_079_128.pdf',\n",
       " 'eng_08_080_004.pdf',\n",
       " 'eng_08_080_012.pdf',\n",
       " 'eng_08_080_016.pdf',\n",
       " 'eng_08_080_027.pdf',\n",
       " 'eng_08_080_031.pdf',\n",
       " 'eng_08_080_040.pdf',\n",
       " 'eng_08_080_046.pdf',\n",
       " 'eng_08_080_069.pdf',\n",
       " 'eng_08_081_001.pdf',\n",
       " 'eng_08_081_002.pdf',\n",
       " 'eng_08_081_018.pdf',\n",
       " 'eng_08_081_045.pdf',\n",
       " 'eng_08_081_056.pdf',\n",
       " 'eng_08_081_075.pdf',\n",
       " 'eng_08_081_076.pdf',\n",
       " 'eng_08_081_123.pdf',\n",
       " 'eng_08_081_124.pdf',\n",
       " 'eng_08_082_004.pdf',\n",
       " 'eng_08_082_006.pdf',\n",
       " 'eng_08_082_025.pdf',\n",
       " 'eng_08_082_035.pdf',\n",
       " 'eng_08_082_036.pdf',\n",
       " 'eng_08_082_083.pdf',\n",
       " 'eng_08_082_085.pdf',\n",
       " 'eng_08_083_041.pdf',\n",
       " 'eng_08_083_042.pdf',\n",
       " 'eng_08_083_046.pdf',\n",
       " 'eng_08_083_074.pdf',\n",
       " 'eng_08_083_124.pdf',\n",
       " 'eng_08_083_135.pdf',\n",
       " 'eng_08_084_036.pdf',\n",
       " 'eng_08_084_107.pdf',\n",
       " 'eng_08_084_138.pdf',\n",
       " 'eng_08_084_177.pdf',\n",
       " 'eng_08_085_013.pdf',\n",
       " 'eng_08_085_014.pdf',\n",
       " 'eng_08_085_055.pdf',\n",
       " 'eng_08_085_059.pdf',\n",
       " 'eng_08_085_107.pdf',\n",
       " 'eng_08_085_120.pdf',\n",
       " 'eng_08_085_137.pdf',\n",
       " 'eng_08_085_145.pdf',\n",
       " 'eng_08_086_009.pdf',\n",
       " 'eng_08_086_019.pdf',\n",
       " 'eng_08_086_037.pdf',\n",
       " 'eng_08_086_047.pdf',\n",
       " 'eng_08_086_084.pdf',\n",
       " 'eng_08_086_094.pdf',\n",
       " 'eng_08_086_126.pdf',\n",
       " 'eng_08_086_128.pdf',\n",
       " 'eng_08_087_062.pdf',\n",
       " 'eng_08_087_108.pdf',\n",
       " 'eng_08_087_141.pdf',\n",
       " 'eng_08_087_155.pdf',\n",
       " 'eng_09_088_008.pdf',\n",
       " 'eng_09_088_009.pdf',\n",
       " 'eng_09_088_014.pdf',\n",
       " 'eng_09_088_025.pdf',\n",
       " 'eng_09_088_053.pdf',\n",
       " 'eng_09_088_067.pdf',\n",
       " 'eng_09_088_080.pdf',\n",
       " 'eng_09_088_100.pdf',\n",
       " 'eng_09_088_107.pdf',\n",
       " 'eng_09_088_108.pdf',\n",
       " 'eng_09_088_119.pdf',\n",
       " 'eng_09_088_184.pdf',\n",
       " 'eng_09_089_033.pdf',\n",
       " 'eng_09_089_035.pdf',\n",
       " 'eng_09_089_077.pdf',\n",
       " 'eng_09_089_091.pdf',\n",
       " 'eng_09_089_102.pdf',\n",
       " 'eng_09_089_125.pdf',\n",
       " 'eng_09_089_128.pdf',\n",
       " 'eng_09_089_139.pdf',\n",
       " 'eng_09_089_154.pdf',\n",
       " 'eng_09_090_022.pdf',\n",
       " 'eng_09_090_035.pdf',\n",
       " 'eng_09_090_037.pdf',\n",
       " 'eng_09_090_054.pdf',\n",
       " 'eng_09_090_108.pdf',\n",
       " 'eng_09_090_134.pdf',\n",
       " 'eng_09_090_164.pdf',\n",
       " 'eng_09_090_171.pdf',\n",
       " 'eng_09_090_182.pdf',\n",
       " 'eng_09_091_027.pdf',\n",
       " 'eng_09_091_029.pdf',\n",
       " 'eng_09_091_055.pdf',\n",
       " 'eng_09_091_094.pdf',\n",
       " 'eng_09_091_128.pdf',\n",
       " 'eng_09_091_130.pdf',\n",
       " 'eng_09_091_163.pdf',\n",
       " 'eng_09_091_178.pdf',\n",
       " 'eng_09_092_002.pdf',\n",
       " 'eng_09_092_003.pdf',\n",
       " 'eng_09_092_011.pdf',\n",
       " 'eng_09_092_084.pdf',\n",
       " 'eng_09_092_096.pdf',\n",
       " 'eng_09_092_102.pdf',\n",
       " 'eng_09_092_105.pdf',\n",
       " 'eng_09_092_113.pdf',\n",
       " 'eng_09_092_115.pdf',\n",
       " 'eng_09_092_132.pdf',\n",
       " 'eng_09_092_142.pdf',\n",
       " 'eng_09_092_154.pdf',\n",
       " 'eng_09_092_182.pdf',\n",
       " 'eng_10_093_001.pdf',\n",
       " 'eng_10_093_009.pdf',\n",
       " 'eng_10_093_020.pdf',\n",
       " 'eng_10_093_031.pdf',\n",
       " 'eng_10_093_038.pdf',\n",
       " 'eng_10_093_044.pdf',\n",
       " 'eng_10_093_070.pdf',\n",
       " 'eng_10_093_100.pdf',\n",
       " 'eng_10_093_124.pdf',\n",
       " 'eng_10_093_156.pdf',\n",
       " 'eng_10_094_016.pdf',\n",
       " 'eng_10_094_027.pdf',\n",
       " 'eng_10_094_035.pdf',\n",
       " 'eng_10_094_047.pdf',\n",
       " 'eng_10_094_067.pdf',\n",
       " 'eng_10_094_091.pdf',\n",
       " 'eng_10_094_094.pdf',\n",
       " 'eng_10_094_105.pdf',\n",
       " 'eng_10_094_164.pdf',\n",
       " 'eng_10_095_016.pdf',\n",
       " 'eng_10_095_030.pdf',\n",
       " 'eng_10_095_046.pdf',\n",
       " 'eng_10_095_058.pdf',\n",
       " 'eng_10_095_068.pdf',\n",
       " 'eng_10_095_089.pdf',\n",
       " 'eng_10_095_100.pdf',\n",
       " 'eng_10_095_120.pdf',\n",
       " 'eng_10_095_123.pdf',\n",
       " 'eng_10_095_133.pdf',\n",
       " 'eng_10_096_008.pdf',\n",
       " 'eng_10_096_017.pdf',\n",
       " 'eng_10_096_051.pdf',\n",
       " 'eng_10_096_066.pdf',\n",
       " 'eng_10_096_087.pdf',\n",
       " 'eng_10_096_095.pdf',\n",
       " 'eng_10_096_120.pdf',\n",
       " 'eng_10_096_136.pdf',\n",
       " 'eng_10_096_146.pdf',\n",
       " 'eng_10_097_053.pdf',\n",
       " 'eng_10_097_069.pdf',\n",
       " 'eng_10_097_072.pdf',\n",
       " 'eng_10_097_096.pdf',\n",
       " 'eng_10_097_105.pdf',\n",
       " 'eng_10_097_107.pdf',\n",
       " 'eng_10_097_112.pdf',\n",
       " 'eng_10_097_162.pdf',\n",
       " 'eng_10_097_166.pdf',\n",
       " 'eng_10_098_004.pdf',\n",
       " 'eng_10_098_007.pdf',\n",
       " 'eng_10_098_014.pdf',\n",
       " 'eng_10_098_028.pdf',\n",
       " 'eng_10_098_047.pdf',\n",
       " 'eng_10_098_049.pdf',\n",
       " 'eng_10_098_082.pdf',\n",
       " 'eng_10_098_105.pdf',\n",
       " 'eng_10_098_113.pdf',\n",
       " 'eng_10_098_115.pdf',\n",
       " 'eng_10_098_124.pdf']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sort_nicely(state_pdfs_files)\n",
    "state_pdfs_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_4_numbers(crop_stat_path):\n",
    "    \n",
    "    text = (pytesseract.image_to_string(crop_stat_path, config='--psm 6', lang='eng')) #config='--psm 4' config='-c preserve_interword_spaces=1'\n",
    "    \n",
    "    text = re.findall(r'\\d+', text)\n",
    "        \n",
    "    if len(text)==3:\n",
    "        if int(text[0]) + int(text[1]) == int(text[2]):\n",
    "            net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = text[0],text[1],\"0\",text[2]\n",
    "        else:\n",
    "            net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = \"\",\"\",\"\",\"\"\n",
    "    else:\n",
    "        net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = \"\",\"\",\"\",\"\"\n",
    "    \n",
    "    return net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total\n",
    "\n",
    "def split_data(data, seps):\n",
    "    \n",
    "    for s in seps:\n",
    "        \n",
    "        if s in data:\n",
    "            break\n",
    "    \n",
    "    data = data.split(s)\n",
    "\n",
    "    if len(data)>1:\n",
    "        data = data[-1].strip()\n",
    "        return data\n",
    "    else:\n",
    "        data = \"\"\n",
    "        return data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_stats(a,b,c,d,img,crop_path,count):\n",
    "\n",
    "    crop_img = crop_section(a,b,c,d,img)\n",
    "\n",
    "    crop_path = input_images_blocks_path+\"page/\"\n",
    "    create_path(crop_path)\n",
    "\n",
    "    crop_stat_path = crop_path+\"stat.jpg\"\n",
    "    crop_img.save(crop_stat_path)\n",
    "\n",
    "    a_n,b_n,c_n,d_n = extract_4_numbers(crop_stat_path)\n",
    "    \n",
    "    if count == 100:\n",
    "        return a_n,b_n,c_n,d_n\n",
    "    \n",
    "    elif a_n == \"\" and b_n == \"\":\n",
    "        a_n,b_n,c_n,d_n = get_stats(a,b-20,c,d,img,crop_path,count+1)\n",
    "    else:\n",
    "        return a_n,b_n,c_n,d_n\n",
    "    \n",
    "    return a_n,b_n,c_n,d_n\n",
    "\n",
    "def get_police_det(a,b,c,d,img,crop_path):\n",
    "\n",
    "    crop_img = crop_section(a,b,c,d,img)\n",
    "\n",
    "    crop_police_path = crop_path+\"police.jpg\"\n",
    "    crop_img.save(crop_police_path)\n",
    "\n",
    "    text = (pytesseract.image_to_string(crop_police_path, config='--psm 6', lang='eng')) #config='--psm 4' config='-c preserve_interword_spaces=1'\n",
    "    text = text.split('\\n')\n",
    "    text = [ i for i in text if i!='' and i!='\\x0c']\n",
    "    \n",
    "    polling_station_name, polling_station_address = \"\",\"\"\n",
    "    \n",
    "    \n",
    "    if len(text) == 2:\n",
    "        polling_station_name = text[1].strip()\n",
    "    elif len(text) == 3:\n",
    "        if \"Number\" in text[0]:\n",
    "            polling_station_name = text[1] + text[2].strip()\n",
    "        if \"Number\" in text[1]:\n",
    "            polling_station_name = text[2].strip()\n",
    "        \n",
    "    elif len(text) == 4:\n",
    "        \n",
    "        if \"Number\" in text[1]:\n",
    "            polling_station_name = text[2] + text[3].strip()\n",
    "        if \"Number\" in text[2]:\n",
    "            polling_station_name = text[3].strip()\n",
    "\n",
    "    return polling_station_name, polling_station_address\n",
    "\n",
    "def extract_first_page_details(path):\n",
    "\n",
    "    img = Image.open(path)\n",
    "    crop_path = input_images_blocks_path+\"page/\"\n",
    "    create_path(crop_path)\n",
    "    \n",
    "    a,b,c,d = 160,2791,1787,462 # police name name and address\n",
    "    polling_station_name, polling_station_address = get_police_det(a,b,c,d,img,crop_path)\n",
    "    \n",
    "    \n",
    "    a,b,c,d = 3136,1570,790,960  # mandal block\n",
    "    crop_img = crop_section(a,b,c,d,img)\n",
    "    \n",
    "    crop_det_path = crop_path+\"det.jpg\"\n",
    "    crop_img.save(crop_det_path)\n",
    "\n",
    "    text = (pytesseract.image_to_string(crop_det_path, config='--psm 6', lang='eng+hin')) #config='--psm 4' config='-c preserve_interword_spaces=1'\n",
    "    text = text.split('\\n')\n",
    "    text = [ i for i in text if i!='' and i!='\\x0c']\n",
    "    \n",
    "    main_town,revenue_division,police_station,mandal,district,pin_code = \"\",\"\",\"\",\"\",\"\",\"\"\n",
    "    \n",
    "    \n",
    "    if len(text) == 3:\n",
    "        main_town = text[0].strip()        \n",
    "        mandal = text[1].strip()\n",
    "        district = text[2].strip()\n",
    "    \n",
    "\n",
    "    a,b,c,d = 205,400,2500,570 # ac name and parl\n",
    "    crop_img = crop_section(a,b,c,d,img)\n",
    "    \n",
    "    crop_ac_path = crop_path+\"ac.jpg\"\n",
    "    crop_img.save(crop_ac_path)\n",
    "\n",
    "    text = (pytesseract.image_to_string(crop_ac_path, config='--psm 6', lang='eng')) #config='--psm 4' config='-c preserve_interword_spaces=1'\n",
    "    text = text.split('\\n')\n",
    "    text = [ i for i in text if i!='' and i!='\\x0c']\n",
    "    \n",
    "    ac_name, parl_constituency = \"\",\"\"\n",
    "    \n",
    "    if len(text) == 6:\n",
    "        ac_name = text[1].strip()\n",
    "        ac_name = ac_name.replace(\";\",\"\")\n",
    "        ac_name = ac_name.replace(\".\",\"\")\n",
    "        ac_name = ac_name.replace(\"|\",\"\")\n",
    "        \n",
    "        parl_constituency = text[4].split(\"which\")\n",
    "        if len(parl_constituency)>1:\n",
    "            parl_constituency = parl_constituency[-1].strip()\n",
    "        \n",
    "    \n",
    "    a,b,c,d = 3596,500,403,141 # part no\n",
    "    crop_img = crop_section(a,b,c,d,img)\n",
    "    \n",
    "    crop_part_path = crop_path+\"part.jpg\"\n",
    "    crop_img.save(crop_part_path)\n",
    "\n",
    "    text = (pytesseract.image_to_string(crop_part_path, config='--psm 6', lang='eng')) #config='--psm 4' config='-c preserve_interword_spaces=1'\n",
    "    text = re.findall(r'\\d+', text)\n",
    "    \n",
    "    if len(text)>0:\n",
    "        part_no = text[0]\n",
    "    else:\n",
    "        part_no = \"\"\n",
    "    \n",
    "    \n",
    "        \n",
    "    a,b,c,d = 2410,5020,1328,90  # stats for male and female\n",
    "    a_n,b_n,c_n,d_n = get_stats(a,b,c,d,img,crop_path,0)\n",
    "\n",
    "    \n",
    "    return [ac_name,parl_constituency,part_no,main_town,police_station,polling_station_name,polling_station_address,revenue_division,mandal,district,pin_code,a_n,b_n,c_n,d_n]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_poll_blocks_from_page(page_full_path,page_blocks_path,amend_page):\n",
    "    \n",
    "    img = Image.open(page_full_path)\n",
    "\n",
    "    amend = False\n",
    "    \n",
    "    def generate(intial_width,a,b,gap):\n",
    "        count = 0\n",
    "        crop_width = 1350\n",
    "        crop_height = 430\n",
    "\n",
    "        for col in range(1,11):\n",
    "\n",
    "            for row in range(1,4):\n",
    "                c = a+crop_width\n",
    "                d = b+crop_height\n",
    "                area = (a, b, c, d)\n",
    "                cropped_img = img.crop(area)\n",
    "                count = count+1\n",
    "                cropped_img.save(page_blocks_path+str(count)+\".jpg\")\n",
    "\n",
    "                a = c\n",
    "\n",
    "            a = intial_width\n",
    "            b = b+crop_height+gap\n",
    "    \n",
    "    page_type,intial_height = check_page_type(img,amend_page)\n",
    "    \n",
    "    if page_type == 1:\n",
    "        intial_width = 32\n",
    "        generate(intial_width,intial_width,intial_height,-10)\n",
    "    \n",
    "def check_page_type(img,amend_page):\n",
    "    return 1,468\n",
    "    \n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_name(name):\n",
    "    \n",
    "    seps = [\"Name\"]\n",
    "    name = split_data(name,seps)\n",
    "    \n",
    "    return name\n",
    "    \n",
    "def extract_vid(v_id):\n",
    "    row = v_id.split(\"|\")\n",
    "    \n",
    "    if len(row)>=2:\n",
    "        number = re.findall(r'\\d+', row[0].strip())\n",
    "        if len(number)>0:\n",
    "            return number[0],row[-1]\n",
    "        else:\n",
    "            return \"\",row[-1]\n",
    "    \n",
    "    return \"\", v_id\n",
    "\n",
    "def extract_house_no(house_no):\n",
    "    seps = [\"Number\"]\n",
    "    house_no = split_data(house_no,seps)\n",
    "    return house_no\n",
    "    \n",
    "def extract_age_gender(age_gender):\n",
    "    \n",
    "    age = re.findall(r'\\d+', age_gender.strip())\n",
    "    if len(age)>0:\n",
    "        age = age[0]\n",
    "    \n",
    "    gender = ''\n",
    "    \n",
    "    seps = [\"Age\"]\n",
    "    \n",
    "    age_l = split_data(age_gender, seps)\n",
    "    \n",
    "    if 'F' in age_l.strip():\n",
    "        gender = 'Female'\n",
    "    elif 'M' in age_l.strip():\n",
    "        gender = \"Male\"\n",
    "    else:\n",
    "        gender = \"\"\n",
    "\n",
    "    return age, gender\n",
    "\n",
    "def extract_rel_name(rel_name):\n",
    "    \n",
    "    seps = [\"Name\"]\n",
    "    \n",
    "    name = split_data(rel_name, seps)\n",
    "    rel_type = extract_rel_type(rel_name)\n",
    "           \n",
    "    return name,rel_type\n",
    "    \n",
    "def extract_rel_type(rel_type):\n",
    "\n",
    "    if \"Father\" in rel_type:\n",
    "        rel_type = 'father'\n",
    "        \n",
    "    elif \"Husband\" in rel_type:\n",
    "        rel_type = 'husband'\n",
    "        \n",
    "    elif \"Mother\" in rel_type:\n",
    "        rel_type = 'mother'\n",
    "        \n",
    "    else:\n",
    "        rel_type = \"father\"\n",
    "    \n",
    "    return rel_type\n",
    "\n",
    "\n",
    "def extract_details_from_block(block):\n",
    "    \n",
    "    if len(block)==6:\n",
    "    \n",
    "        v_id = block[0]\n",
    "        name = block[1]\n",
    "        rel_name = block[2]\n",
    "        house_no = block[3]\n",
    "        age_gender = block[5]\n",
    "    \n",
    "    name = extract_name(name)\n",
    "    rel_name,rel_type = extract_rel_name(rel_name)\n",
    "    house_no = extract_house_no(house_no)\n",
    "    age, gender = extract_age_gender(age_gender)\n",
    "    number,voter_id = extract_vid(v_id)\n",
    "    \n",
    "    return [name,rel_name,rel_type,house_no,age,gender,voter_id,number]\n",
    "\n",
    "def arrange_columns(first_page_list,block_list,filename):\n",
    "    \n",
    "    year = 2018\n",
    "    state = 'kerala'\n",
    "    \n",
    "    ac_name,parl_constituency,part_no,main_town,police_station,polling_station_name,polling_station_address,revenue_division,mandal,district,pin_code,net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = first_page_list\n",
    "    name,rel_name,rel_type,house_no,age,gender,voter_id,number = block_list\n",
    "    \n",
    "    final_list = [number,voter_id,name,rel_name,rel_type,house_no,age,gender,ac_name,\n",
    "                 parl_constituency,part_no,year,state,filename,main_town,police_station,mandal,\n",
    "                 revenue_division,district,pin_code,polling_station_name,polling_station_address,\n",
    "                 net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total]\n",
    "    \n",
    "    \n",
    "    return final_list"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "print(name)\n",
    "print(rel_name,rel_type)\n",
    "print(house_no)\n",
    "print(age,gender)\n",
    "print(voter_id)\n",
    "print()\n",
    "\n",
    "\n",
    "print(ac_name)\n",
    "print(parl_constituency)\n",
    "print(part_no)\n",
    "print(main_town)\n",
    "print(mandal)\n",
    "print(district)\n",
    "print(polling_station_name)\n",
    "print(a_n,b_n,c_n,d_n)\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "eng_01_001_011.pdf\n",
      "page done 3.jpg\n",
      "page done 4.jpg\n",
      "page done 5.jpg\n",
      "page done 6.jpg\n",
      "page done 7.jpg\n",
      "page done 8.jpg\n",
      "page done 9.jpg\n",
      "page done 10.jpg\n",
      "page done 11.jpg\n",
      "page done 12.jpg\n",
      "page done 13.jpg\n",
      "page done 14.jpg\n",
      "page done 15.jpg\n",
      "page done 16.jpg\n",
      "page done 17.jpg\n",
      "page done 18.jpg\n",
      "page done 19.jpg\n",
      "page done 20.jpg\n",
      "page done 21.jpg\n",
      "page done 22.jpg\n",
      "page done 23.jpg\n",
      "page done 24.jpg\n",
      "page done 25.jpg\n",
      "page done 26.jpg\n",
      "page done 27.jpg\n",
      "CSV saved\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# for pdf_file_name in ['eng_01_001_011.pdf']:\n",
    "for pdf_file_name in state_pdfs_files:\n",
    "\n",
    "    print(pdf_file_name)\n",
    "    \n",
    "    if pdf_file_name == \".DS_Store\":\n",
    "        continue\n",
    "    \n",
    "    if not pdf_file_name.endswith(\".pdf\"):\n",
    "        continue\n",
    "    \n",
    "    #create images,blocks and csvs paths for each file\n",
    "    pdf_file_name_without_ext = pdf_file_name.split('.pdf')[0]\n",
    "    input_pdf_images_path = PARSE_DATA_PAGES+pdf_file_name_without_ext+\"/\"\n",
    "    create_path(input_pdf_images_path)\n",
    "    \n",
    "    input_images_blocks_path = PARSE_DATA_BLOCKS+pdf_file_name_without_ext+\"/\"\n",
    "    create_path(input_images_blocks_path)\n",
    "    \n",
    "    if os.path.exists(PARSE_DATA_CSVS+pdf_file_name_without_ext+\".csv\"):\n",
    "        print(pdf_file_name_without_ext+\".csv\", \"already exists\")\n",
    "        continue\n",
    "    \n",
    "    #convert pdf into bunch of images\n",
    "    try:\n",
    "        pdf_2_images_list = pdf_to_img(state_pdfs_path+pdf_file_name, input_pdf_images_path,dpi=500)\n",
    "    except:\n",
    "        continue\n",
    "    \n",
    "    \n",
    "    #sort pages for looping\n",
    "    input_images = os.listdir(input_pdf_images_path)\n",
    "    sort_nicely(input_images)\n",
    "    \n",
    "    #empty intial data\n",
    "    df = pd.DataFrame(columns = COLUMNS)\n",
    "    order_problem = []\n",
    "        \n",
    "    amend_page = False\n",
    "    \n",
    "           \n",
    "    #for each page, parse the data\n",
    "    for page in input_images:\n",
    "        \n",
    "        page_full_path = input_pdf_images_path+page\n",
    "        \n",
    "        #extract first page content\n",
    "        if page == '1.jpg':\n",
    "            first_page_list = extract_first_page_details(page_full_path)\n",
    "            continue\n",
    "\n",
    "        #ingnore 2nd page and last page\n",
    "        if page == '2.jpg' or input_images[-1] == page:\n",
    "            continue\n",
    "\n",
    "\n",
    "        #loop from 3 page onwards\n",
    "        if page.endswith('.jpg'):\n",
    "            \n",
    "            \n",
    "            final_invidual_blocks = []\n",
    "            blocks_path = input_images_blocks_path+\"blocks/\"\n",
    "            create_path(blocks_path)\n",
    "\n",
    "            page_idx = page.split(\".jpg\")[0] + \"/\"\n",
    "            page_blocks_path = blocks_path+page_idx\n",
    "            create_path(page_blocks_path)\n",
    "                        \n",
    "            generate_poll_blocks_from_page(page_full_path,page_blocks_path,amend_page)\n",
    "            \n",
    "            \n",
    "            sorted_blocks = os.listdir(page_blocks_path)\n",
    "            sort_nicely(sorted_blocks)\n",
    "            \n",
    "            for jpg_file in sorted_blocks:\n",
    "                \n",
    "                if jpg_file.endswith('.jpg') :\n",
    "\n",
    "                    text = (pytesseract.image_to_string(page_blocks_path+jpg_file, config='--psm 6', lang='eng')) #config='--psm 4' config='-c preserve_interword_spaces=1'\n",
    "                    params_list = text.split('\\n')\n",
    "                    new_params_list = [ i for i in params_list if i!='' and i!='\\x0c']\n",
    "                    \n",
    "                    if len(new_params_list)==6:\n",
    "                        final_invidual_blocks.append(new_params_list)\n",
    "                    else:\n",
    "                        order_problem.append((page, jpg_file,new_params_list))\n",
    "\n",
    "        #put the data into dataframe\n",
    "        for block in final_invidual_blocks:\n",
    "            block_list = extract_details_from_block(block)\n",
    "            final_list = arrange_columns(first_page_list,block_list,pdf_file_name_without_ext)\n",
    "\n",
    "            df_length = len(df)\n",
    "            df.loc[df_length] = final_list\n",
    "            \n",
    "        print(\"page done\", page)\n",
    "                \n",
    "    #save the dataframe(pdf) data into csv\n",
    "    save_to_csv(df,PARSE_DATA_CSVS+pdf_file_name_without_ext+\".csv\")\n",
    "    print(\"CSV saved\")\n",
    "    print()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#combine all state files into one csv\n",
    "combine_all_csvs(\"keral_final.csv\",PARSE_DATA_CSVS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}