scripts/tamil_nadu/parse_tamilnadu_tamil.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pdf2image\n",
    "from PIL import Image\n",
    "import pytesseract\n",
    "import difflib\n",
    "import re\n",
    "import pandas as pd\n",
    "import sys\n",
    "sys.path.append('../')\n",
    "from helper import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = '../../data/'\n",
    "STATE = 'tamilnadu/'\n",
    "\n",
    "PARSE_DATA_PAGES = \"../../parseData/images/\"+STATE\n",
    "create_path(PARSE_DATA_PAGES)\n",
    "\n",
    "PARSE_DATA_BLOCKS = \"../../parseData/blocks/\"+STATE\n",
    "create_path(PARSE_DATA_BLOCKS)\n",
    "\n",
    "PARSE_DATA_CSVS = \"../../parseData/csvs/\"+STATE\n",
    "create_path(PARSE_DATA_CSVS)\n",
    "\n",
    "COLUMNS = [\"number\",\"id\", \"elector_name\", \"father_or_husband_name\", \"relationship\", \"house_no\", \"age\", \"sex\", \"ac_name\", \"parl_constituency\", \"part_no\", \"year\", \"state\", \"filename\", \"main_town\", \"police_station\", \"mandal\", \"revenue_division\", \"district\", \"pin_code\", \"polling_station_name\", \"polling_station_address\", \"net_electors_male\", \"net_electors_female\", \"net_electors_third_gender\", \"net_electors_total\"]\n",
    "\n",
    "state_pdfs_path = DATA_PATH+STATE\n",
    "state_pdfs_files = os.listdir(state_pdfs_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['.DS_Store',\n",
       " 'dt1_ac001_ac001126.pdf',\n",
       " 'dt1_ac001_ac001133.pdf',\n",
       " 'dt1_ac001_ac001140.pdf',\n",
       " 'dt1_ac002_ac002082.pdf',\n",
       " 'dt1_ac002_ac002099.pdf',\n",
       " 'dt1_ac002_ac002270.pdf',\n",
       " 'dt1_ac002_ac002287.pdf',\n",
       " 'dt1_ac003_ac003019.pdf',\n",
       " 'dt1_ac003_ac003026.pdf',\n",
       " 'dt1_ac003_ac003033.pdf',\n",
       " 'dt1_ac003_ac003214.pdf',\n",
       " 'dt1_ac003_ac003221.pdf',\n",
       " 'dt1_ac003_ac003238.pdf',\n",
       " 'dt1_ac004_ac004163.pdf',\n",
       " 'dt1_ac004_ac004170.pdf',\n",
       " 'dt1_ac004_ac004187.pdf',\n",
       " 'dt1_ac005_ac005107.pdf',\n",
       " 'dt1_ac005_ac005114.pdf',\n",
       " 'dt1_ac005_ac005121.pdf',\n",
       " 'dt1_ac005_ac005302.pdf',\n",
       " 'dt1_ac006_ac006056.pdf',\n",
       " 'dt1_ac006_ac006063.pdf',\n",
       " 'dt1_ac006_ac006070.pdf',\n",
       " 'dt1_ac006_ac006244.pdf',\n",
       " 'dt1_ac006_ac006251.pdf',\n",
       " 'dt1_ac007_ac007007.pdf',\n",
       " 'dt1_ac007_ac007014.pdf',\n",
       " 'dt1_ac007_ac007193.pdf',\n",
       " 'dt1_ac008_ac008137.pdf',\n",
       " 'dt1_ac008_ac008144.pdf',\n",
       " 'dt1_ac008_ac008151.pdf',\n",
       " 'dt1_ac009_ac009086.pdf',\n",
       " 'dt1_ac009_ac009093.pdf',\n",
       " 'dt1_ac009_ac009274.pdf',\n",
       " 'dt1_ac009_ac009281.pdf',\n",
       " 'dt1_ac009_ac009298.pdf',\n",
       " 'dt1_ac010_ac010110.pdf',\n",
       " 'dt1_ac010_ac010127.pdf',\n",
       " 'dt2_ac011_ac011146.pdf',\n",
       " 'dt2_ac011_ac011153.pdf',\n",
       " 'dt2_ac011_ac011160.pdf',\n",
       " 'dt2_ac012_ac012283.pdf',\n",
       " 'dt2_ac013_ac013039.pdf',\n",
       " 'dt2_ac013_ac013046.pdf',\n",
       " 'dt2_ac014_ac014176.pdf',\n",
       " 'dt2_ac014_ac014183.pdf',\n",
       " 'dt2_ac014_ac014190.pdf',\n",
       " 'dt2_ac015_ac015110.pdf',\n",
       " 'dt2_ac015_ac015127.pdf',\n",
       " 'dt2_ac015_ac015134.pdf',\n",
       " 'dt2_ac016_ac016069.pdf',\n",
       " 'dt2_ac016_ac016076.pdf',\n",
       " 'dt2_ac016_ac016083.pdf',\n",
       " 'dt2_ac017_ac017003.pdf',\n",
       " 'dt2_ac017_ac017010.pdf',\n",
       " 'dt2_ac017_ac017027.pdf',\n",
       " 'dt2_ac018_ac018140.pdf',\n",
       " 'dt2_ac018_ac018157.pdf',\n",
       " 'dt2_ac018_ac018164.pdf',\n",
       " 'dt2_ac019_ac019108.pdf',\n",
       " 'dt2_ac020_ac020130.pdf',\n",
       " 'dt2_ac020_ac020147.pdf',\n",
       " 'dt2_ac021_ac021089.pdf',\n",
       " 'dt2_ac021_ac021096.pdf',\n",
       " 'dt2_ac022_ac022016.pdf',\n",
       " 'dt2_ac022_ac022023.pdf',\n",
       " 'dt2_ac022_ac022030.pdf',\n",
       " 'dt2_ac022_ac022204.pdf',\n",
       " 'dt2_ac022_ac022211.pdf',\n",
       " 'dt2_ac022_ac022228.pdf',\n",
       " 'dt2_ac023_ac023153.pdf',\n",
       " 'dt2_ac023_ac023160.pdf',\n",
       " 'dt2_ac023_ac023177.pdf',\n",
       " 'dt2_ac024_ac024104.pdf',\n",
       " 'dt2_ac024_ac024111.pdf',\n",
       " 'dt2_ac025_ac025046.pdf',\n",
       " 'dt2_ac025_ac025053.pdf',\n",
       " 'dt2_ac025_ac025060.pdf',\n",
       " 'dt2_ac025_ac025234.pdf',\n",
       " 'dt2_ac025_ac025241.pdf',\n",
       " 'dt2_ac026_ac026004.pdf',\n",
       " 'dt2_ac026_ac026190.pdf',\n",
       " 'dt3_ac027_ac027016.pdf',\n",
       " 'dt3_ac027_ac027023.pdf',\n",
       " 'dt3_ac027_ac027030.pdf',\n",
       " 'dt3_ac027_ac027204.pdf',\n",
       " 'dt3_ac027_ac027211.pdf',\n",
       " 'dt3_ac027_ac027228.pdf',\n",
       " 'dt3_ac027_ac027409.pdf',\n",
       " 'dt3_ac027_ac027416.pdf',\n",
       " 'dt3_ac028_ac028160.pdf',\n",
       " 'dt3_ac028_ac028177.pdf',\n",
       " 'dt3_ac028_ac028358.pdf',\n",
       " 'dt3_ac028_ac028365.pdf',\n",
       " 'dt3_ac028_ac028372.pdf',\n",
       " 'dt3_ac029_ac029104.pdf',\n",
       " 'dt3_ac029_ac029111.pdf',\n",
       " 'dt3_ac029_ac029309.pdf',\n",
       " 'dt3_ac030_ac030143.pdf',\n",
       " 'dt3_ac030_ac030150.pdf',\n",
       " 'dt3_ac030_ac030167.pdf',\n",
       " 'dt3_ac030_ac030324.pdf',\n",
       " 'dt3_ac030_ac030331.pdf',\n",
       " 'dt3_ac030_ac030348.pdf',\n",
       " 'dt3_ac031_ac031092.pdf',\n",
       " 'dt3_ac031_ac031280.pdf',\n",
       " 'dt3_ac031_ac031297.pdf',\n",
       " 'dt3_ac032_ac032036.pdf',\n",
       " 'dt3_ac032_ac032043.pdf',\n",
       " 'dt3_ac032_ac032050.pdf',\n",
       " 'dt3_ac032_ac032217.pdf',\n",
       " 'dt3_ac032_ac032224.pdf',\n",
       " 'dt3_ac032_ac032231.pdf',\n",
       " 'dt3_ac032_ac032405.pdf',\n",
       " 'dt3_ac032_ac032412.pdf',\n",
       " 'dt3_ac033_ac033166.pdf',\n",
       " 'dt3_ac033_ac033173.pdf',\n",
       " 'dt3_ac033_ac033180.pdf',\n",
       " 'dt3_ac034_ac034117.pdf',\n",
       " 'dt3_ac034_ac034124.pdf',\n",
       " 'dt3_ac035_ac035059.pdf',\n",
       " 'dt3_ac035_ac035066.pdf',\n",
       " 'dt3_ac035_ac035073.pdf',\n",
       " 'dt3_ac035_ac035254.pdf',\n",
       " 'dt3_ac035_ac035261.pdf',\n",
       " 'dt3_ac036_ac036017.pdf',\n",
       " 'dt3_ac036_ac036205.pdf',\n",
       " 'dt3_ac037_ac037147.pdf',\n",
       " 'dt3_ac037_ac037154.pdf',\n",
       " 'dt3_ac037_ac037161.pdf',\n",
       " 'dt4_ac038_ac038173.pdf',\n",
       " 'dt4_ac038_ac038180.pdf',\n",
       " 'dt4_ac038_ac038197.pdf',\n",
       " 'dt4_ac039_ac039117.pdf',\n",
       " 'dt4_ac039_ac039124.pdf',\n",
       " 'dt4_ac040_ac040156.pdf',\n",
       " 'dt4_ac040_ac040163.pdf',\n",
       " 'dt4_ac040_ac040170.pdf',\n",
       " 'dt4_ac041_ac041107.pdf',\n",
       " 'dt4_ac042_ac042049.pdf',\n",
       " 'dt4_ac042_ac042056.pdf',\n",
       " 'dt4_ac042_ac042063.pdf',\n",
       " 'dt4_ac042_ac042237.pdf',\n",
       " 'dt4_ac042_ac042244.pdf',\n",
       " 'dt4_ac043_ac043007.pdf',\n",
       " 'dt4_ac043_ac043186.pdf',\n",
       " 'dt4_ac043_ac043193.pdf',\n",
       " 'dt4_ac044_ac044120.pdf',\n",
       " 'dt4_ac044_ac044137.pdf',\n",
       " 'dt4_ac045_ac045079.pdf',\n",
       " 'dt4_ac045_ac045086.pdf',\n",
       " 'dt4_ac046_ac046013.pdf',\n",
       " 'dt4_ac046_ac046020.pdf',\n",
       " 'dt4_ac046_ac046201.pdf',\n",
       " 'dt4_ac046_ac046218.pdf',\n",
       " 'dt4_ac046_ac046225.pdf',\n",
       " 'dt4_ac047_ac047150.pdf',\n",
       " 'dt4_ac047_ac047167.pdf',\n",
       " 'dt4_ac047_ac047174.pdf',\n",
       " 'dt4_ac048_ac048101.pdf',\n",
       " 'dt4_ac048_ac048118.pdf',\n",
       " 'dt4_ac049_ac049043.pdf',\n",
       " 'dt4_ac049_ac049050.pdf',\n",
       " 'dt4_ac049_ac049067.pdf',\n",
       " 'dt4_ac049_ac049231.pdf',\n",
       " 'dt4_ac049_ac049248.pdf',\n",
       " 'dt4_ac049_ac049255.pdf',\n",
       " 'dt4_ac050_ac050082.pdf',\n",
       " 'dt4_ac050_ac050099.pdf',\n",
       " 'dt5_ac051_ac051103.pdf',\n",
       " 'dt5_ac051_ac051110.pdf',\n",
       " 'dt5_ac051_ac051127.pdf',\n",
       " 'dt5_ac052_ac052052.pdf',\n",
       " 'dt5_ac052_ac052069.pdf',\n",
       " 'dt5_ac052_ac052076.pdf',\n",
       " 'dt5_ac052_ac052240.pdf',\n",
       " 'dt5_ac052_ac052257.pdf',\n",
       " 'dt5_ac053_ac053003.pdf',\n",
       " 'dt5_ac053_ac053010.pdf',\n",
       " 'dt5_ac053_ac053199.pdf',\n",
       " 'dt5_ac054_ac054133.pdf',\n",
       " 'dt5_ac054_ac054140.pdf',\n",
       " 'dt5_ac055_ac055082.pdf',\n",
       " 'dt5_ac055_ac055099.pdf',\n",
       " 'dt5_ac055_ac055287.pdf',\n",
       " 'dt5_ac055_ac055294.pdf',\n",
       " 'dt5_ac056_ac056026.pdf',\n",
       " 'dt5_ac056_ac056033.pdf',\n",
       " 'dt5_ac056_ac056040.pdf',\n",
       " 'dt5_ac056_ac056214.pdf',\n",
       " 'dt5_ac056_ac056221.pdf',\n",
       " 'dt5_ac056_ac056238.pdf',\n",
       " 'dt6_ac057_ac057052.pdf',\n",
       " 'dt6_ac057_ac057069.pdf',\n",
       " 'dt6_ac057_ac057076.pdf',\n",
       " 'dt6_ac058_ac058003.pdf',\n",
       " 'dt6_ac058_ac058010.pdf',\n",
       " 'dt6_ac058_ac058199.pdf',\n",
       " 'dt6_ac058_ac058208.pdf',\n",
       " 'dt6_ac059_ac059140.pdf',\n",
       " 'dt6_ac059_ac059157.pdf',\n",
       " 'dt6_ac060_ac060172.pdf',\n",
       " 'dt6_ac060_ac060189.pdf',\n",
       " 'dt6_ac060_ac060196.pdf',\n",
       " 'dt6_ac061_ac061116.pdf',\n",
       " 'dt6_ac061_ac061123.pdf',\n",
       " 'dt6_ac061_ac061130.pdf',\n",
       " 'dt7_ac062_ac062142.pdf',\n",
       " 'dt7_ac062_ac062159.pdf',\n",
       " 'dt7_ac062_ac062166.pdf',\n",
       " 'dt7_ac063_ac063100.pdf',\n",
       " 'dt7_ac063_ac063289.pdf',\n",
       " 'dt7_ac064_ac064035.pdf',\n",
       " 'dt7_ac064_ac064042.pdf',\n",
       " 'dt7_ac064_ac064059.pdf',\n",
       " 'dt7_ac064_ac064223.pdf',\n",
       " 'dt7_ac064_ac064230.pdf',\n",
       " 'dt7_ac064_ac064247.pdf',\n",
       " 'dt7_ac065_ac065172.pdf',\n",
       " 'dt7_ac065_ac065189.pdf',\n",
       " 'dt7_ac065_ac065196.pdf',\n",
       " 'dt7_ac066_ac066116.pdf',\n",
       " 'dt7_ac066_ac066123.pdf',\n",
       " 'dt7_ac066_ac066130.pdf',\n",
       " 'dt7_ac067_ac067065.pdf',\n",
       " 'dt7_ac067_ac067072.pdf',\n",
       " 'dt7_ac067_ac067089.pdf',\n",
       " 'dt7_ac067_ac067260.pdf',\n",
       " 'dt7_ac067_ac067277.pdf',\n",
       " 'dt7_ac068_ac068009.pdf',\n",
       " 'dt7_ac068_ac068016.pdf',\n",
       " 'dt7_ac068_ac068023.pdf',\n",
       " 'dt7_ac068_ac068204.pdf',\n",
       " 'dt7_ac068_ac068211.pdf',\n",
       " 'dt7_ac068_ac068235.pdf',\n",
       " 'dt7_ac069_ac069153.pdf',\n",
       " 'dt7_ac069_ac069160.pdf',\n",
       " 'dt7_ac069_ac069177.pdf',\n",
       " 'dt8_ac070_ac070081.pdf',\n",
       " 'dt8_ac070_ac070098.pdf',\n",
       " 'dt8_ac070_ac070279.pdf',\n",
       " 'dt8_ac070_ac070286.pdf',\n",
       " 'dt8_ac071_ac071018.pdf',\n",
       " 'dt8_ac071_ac071025.pdf',\n",
       " 'dt8_ac071_ac071032.pdf',\n",
       " 'dt8_ac071_ac071206.pdf',\n",
       " 'dt8_ac071_ac071213.pdf',\n",
       " 'dt8_ac071_ac071220.pdf',\n",
       " 'dt8_ac072_ac072162.pdf',\n",
       " 'dt8_ac072_ac072179.pdf',\n",
       " 'dt8_ac073_ac073106.pdf',\n",
       " 'dt8_ac073_ac073113.pdf',\n",
       " 'dt8_ac074_ac074055.pdf',\n",
       " 'dt8_ac074_ac074062.pdf',\n",
       " 'dt8_ac074_ac074236.pdf',\n",
       " 'dt8_ac074_ac074243.pdf',\n",
       " 'dt8_ac074_ac074250.pdf',\n",
       " 'dt8_ac075_ac075006.pdf',\n",
       " 'dt8_ac075_ac075185.pdf',\n",
       " 'dt8_ac075_ac075192.pdf',\n",
       " 'dt8_ac076_ac076129.pdf',\n",
       " 'dt8_ac076_ac076136.pdf',\n",
       " 'dt8_ac076_ac076143.pdf',\n",
       " 'dt8_ac077_ac077078.pdf',\n",
       " 'dt8_ac077_ac077085.pdf',\n",
       " 'dt8_ac077_ac077092.pdf',\n",
       " 'dt8_ac077_ac077273.pdf',\n",
       " 'dt8_ac077_ac077280.pdf',\n",
       " 'dt8_ac077_ac077297.pdf',\n",
       " 'dt8_ac078_ac078012.pdf',\n",
       " 'dt8_ac078_ac078029.pdf',\n",
       " 'dt8_ac078_ac078036.pdf',\n",
       " 'dt8_ac078_ac078217.pdf',\n",
       " 'dt8_ac078_ac078224.pdf',\n",
       " 'dt8_ac078_ac078231.pdf',\n",
       " 'dt8_ac079_ac079166.pdf',\n",
       " 'dt8_ac079_ac079173.pdf',\n",
       " 'dt8_ac079_ac079180.pdf',\n",
       " 'dt8_ac080_ac080002.pdf',\n",
       " 'dt8_ac080_ac080019.pdf',\n",
       " 'dt8_ac080_ac080207.pdf',\n",
       " 'dt9_ac081_ac081038.pdf',\n",
       " 'dt9_ac081_ac081045.pdf',\n",
       " 'dt9_ac081_ac081052.pdf',\n",
       " 'dt9_ac081_ac081226.pdf',\n",
       " 'dt9_ac081_ac081233.pdf',\n",
       " 'dt9_ac081_ac081240.pdf',\n",
       " 'dt9_ac082_ac082175.pdf',\n",
       " 'dt9_ac082_ac082182.pdf',\n",
       " 'dt9_ac083_ac083119.pdf',\n",
       " 'dt9_ac083_ac083126.pdf',\n",
       " 'dt9_ac083_ac083307.pdf',\n",
       " 'dt9_ac084_ac084068.pdf',\n",
       " 'dt9_ac084_ac084075.pdf',\n",
       " 'dt9_ac084_ac084082.pdf',\n",
       " 'dt9_ac084_ac084249.pdf',\n",
       " 'dt9_ac084_ac084256.pdf',\n",
       " 'dt9_ac084_ac084263.pdf',\n",
       " 'dt9_ac085_ac085002.pdf',\n",
       " 'dt9_ac085_ac085019.pdf',\n",
       " 'dt9_ac085_ac085207.pdf',\n",
       " 'dt9_ac086_ac086149.pdf',\n",
       " 'dt9_ac086_ac086156.pdf',\n",
       " 'dt9_ac087_ac087098.pdf',\n",
       " 'dt9_ac087_ac087286.pdf',\n",
       " 'dt9_ac087_ac087293.pdf',\n",
       " 'dt9_ac088_ac088032.pdf',\n",
       " 'dt9_ac088_ac088049.pdf',\n",
       " 'dt9_ac088_ac088220.pdf',\n",
       " 'dt9_ac088_ac088237.pdf',\n",
       " 'dt9_ac088_ac088244.pdf',\n",
       " 'dt9_ac089_ac089179.pdf',\n",
       " 'dt9_ac089_ac089186.pdf',\n",
       " 'dt9_ac089_ac089193.pdf',\n",
       " 'dt9_ac090_ac090015.pdf',\n",
       " 'dt9_ac090_ac090022.pdf',\n",
       " 'dt9_ac090_ac090039.pdf',\n",
       " 'dt9_ac090_ac090203.pdf',\n",
       " 'dt9_ac090_ac090210.pdf',\n",
       " 'dt9_ac091_ac091152.pdf',\n",
       " 'dt9_ac091_ac091169.pdf',\n",
       " 'dt9_ac091_ac091176.pdf',\n",
       " 'dt10_ac092_ac092121.pdf',\n",
       " 'dt10_ac092_ac092138.pdf',\n",
       " 'dt10_ac092_ac092145.pdf',\n",
       " 'dt10_ac093_ac093070.pdf',\n",
       " 'dt10_ac093_ac093087.pdf',\n",
       " 'dt10_ac093_ac093094.pdf',\n",
       " 'dt10_ac093_ac093268.pdf',\n",
       " 'dt10_ac093_ac093275.pdf',\n",
       " 'dt10_ac094_ac094014.pdf',\n",
       " 'dt10_ac094_ac094021.pdf',\n",
       " 'dt10_ac094_ac094202.pdf',\n",
       " 'dt10_ac094_ac094219.pdf',\n",
       " 'dt10_ac095_ac095151.pdf',\n",
       " 'dt10_ac095_ac095168.pdf',\n",
       " 'dt10_ac096_ac096102.pdf',\n",
       " 'dt10_ac097_ac097044.pdf',\n",
       " 'dt10_ac097_ac097051.pdf',\n",
       " 'dt10_ac097_ac097068.pdf',\n",
       " 'dt10_ac097_ac097232.pdf',\n",
       " 'dt10_ac097_ac097249.pdf',\n",
       " 'dt10_ac097_ac097256.pdf',\n",
       " 'dt11_ac098_ac098070.pdf',\n",
       " 'dt11_ac098_ac098087.pdf',\n",
       " 'dt11_ac098_ac098094.pdf',\n",
       " 'dt11_ac099_ac099014.pdf',\n",
       " 'dt11_ac099_ac099021.pdf',\n",
       " 'dt11_ac099_ac099202.pdf',\n",
       " 'dt11_ac099_ac099219.pdf',\n",
       " 'dt11_ac099_ac099226.pdf',\n",
       " 'dt11_ac100_ac100124.pdf',\n",
       " 'dt11_ac100_ac100131.pdf',\n",
       " 'dt11_ac100_ac100148.pdf',\n",
       " 'dt11_ac103_ac103161.pdf',\n",
       " 'dt11_ac103_ac103178.pdf',\n",
       " 'dt11_ac103_ac103185.pdf',\n",
       " 'dt11_ac104_ac104105.pdf',\n",
       " 'dt11_ac104_ac104112.pdf',\n",
       " 'dt11_ac105_ac105054.pdf',\n",
       " 'dt11_ac105_ac105061.pdf',\n",
       " 'dt11_ac105_ac105078.pdf',\n",
       " 'dt11_ac105_ac105242.pdf',\n",
       " 'dt11_ac106_ac106005.pdf',\n",
       " 'dt11_ac106_ac106012.pdf',\n",
       " 'dt11_ac106_ac106191.pdf',\n",
       " 'dt11_ac107_ac107135.pdf',\n",
       " 'dt11_ac107_ac107142.pdf',\n",
       " 'dt12_ac108_ac108161.pdf',\n",
       " 'dt12_ac108_ac108178.pdf',\n",
       " 'dt12_ac108_ac108185.pdf',\n",
       " 'dt12_ac109_ac109105.pdf',\n",
       " 'dt12_ac109_ac109112.pdf',\n",
       " 'dt12_ac110_ac110144.pdf',\n",
       " 'dt12_ac110_ac110151.pdf',\n",
       " 'dt13_ac111_ac111170.pdf',\n",
       " 'dt13_ac111_ac111187.pdf',\n",
       " 'dt13_ac111_ac111194.pdf',\n",
       " 'dt13_ac116_ac116093.pdf',\n",
       " 'dt13_ac116_ac116102.pdf',\n",
       " 'dt13_ac116_ac116281.pdf',\n",
       " 'dt13_ac116_ac116298.pdf',\n",
       " 'dt13_ac117_ac117037.pdf',\n",
       " 'dt13_ac117_ac117044.pdf',\n",
       " 'dt13_ac117_ac117051.pdf',\n",
       " 'dt13_ac117_ac117225.pdf',\n",
       " 'dt13_ac117_ac117232.pdf',\n",
       " 'dt13_ac117_ac117249.pdf',\n",
       " 'dt13_ac118_ac118174.pdf',\n",
       " 'dt13_ac118_ac118181.pdf',\n",
       " 'dt13_ac118_ac118198.pdf',\n",
       " 'dt13_ac119_ac119118.pdf',\n",
       " 'dt13_ac119_ac119125.pdf',\n",
       " 'dt13_ac119_ac119132.pdf',\n",
       " 'dt13_ac120_ac120157.pdf',\n",
       " 'dt13_ac120_ac120164.pdf',\n",
       " 'dt13_ac120_ac120171.pdf',\n",
       " 'dt13_ac121_ac121108.pdf',\n",
       " 'dt13_ac121_ac121294.pdf',\n",
       " 'dt13_ac121_ac121303.pdf',\n",
       " 'dt13_ac122_ac122040.pdf',\n",
       " 'dt13_ac122_ac122057.pdf',\n",
       " 'dt13_ac122_ac122238.pdf',\n",
       " 'dt13_ac122_ac122245.pdf',\n",
       " 'dt13_ac122_ac122252.pdf',\n",
       " 'dt13_ac123_ac123187.pdf',\n",
       " 'dt13_ac123_ac123194.pdf',\n",
       " 'dt13_ac124_ac124121.pdf',\n",
       " 'dt13_ac124_ac124138.pdf',\n",
       " 'dt13_ac124_ac124145.pdf',\n",
       " 'dt14_ac127_ac127057.pdf',\n",
       " 'dt14_ac127_ac127064.pdf',\n",
       " 'dt14_ac127_ac127238.pdf',\n",
       " 'dt14_ac127_ac127245.pdf',\n",
       " 'dt14_ac127_ac127252.pdf',\n",
       " 'dt14_ac128_ac128008.pdf',\n",
       " 'dt14_ac128_ac128187.pdf',\n",
       " 'dt14_ac128_ac128194.pdf',\n",
       " 'dt14_ac129_ac129121.pdf',\n",
       " 'dt14_ac129_ac129138.pdf',\n",
       " 'dt14_ac129_ac129145.pdf',\n",
       " 'dt14_ac130_ac130160.pdf',\n",
       " 'dt14_ac130_ac130177.pdf',\n",
       " 'dt14_ac130_ac130184.pdf',\n",
       " 'dt14_ac131_ac131104.pdf',\n",
       " 'dt14_ac131_ac131111.pdf',\n",
       " 'dt14_ac132_ac132053.pdf',\n",
       " 'dt14_ac132_ac132060.pdf',\n",
       " 'dt14_ac132_ac132077.pdf',\n",
       " 'dt14_ac132_ac132258.pdf',\n",
       " 'dt14_ac132_ac132265.pdf',\n",
       " 'dt14_ac133_ac133004.pdf',\n",
       " 'dt14_ac133_ac133011.pdf',\n",
       " 'dt14_ac133_ac133209.pdf',\n",
       " 'dt15_ac134_ac134023.pdf',\n",
       " 'dt15_ac134_ac134030.pdf',\n",
       " 'dt15_ac134_ac134047.pdf',\n",
       " 'dt15_ac134_ac134228.pdf',\n",
       " 'dt15_ac134_ac134235.pdf',\n",
       " 'dt15_ac135_ac135177.pdf',\n",
       " 'dt15_ac135_ac135184.pdf',\n",
       " 'dt15_ac136_ac136111.pdf',\n",
       " 'dt15_ac136_ac136128.pdf',\n",
       " 'dt15_ac137_ac137060.pdf',\n",
       " 'dt15_ac137_ac137077.pdf',\n",
       " 'dt15_ac137_ac137258.pdf',\n",
       " 'dt16_ac138_ac138096.pdf',\n",
       " 'dt16_ac138_ac138284.pdf',\n",
       " 'dt16_ac138_ac138291.pdf',\n",
       " 'dt16_ac139_ac139030.pdf',\n",
       " 'dt16_ac139_ac139047.pdf',\n",
       " 'dt16_ac139_ac139054.pdf',\n",
       " 'dt16_ac139_ac139228.pdf',\n",
       " 'dt16_ac139_ac139235.pdf',\n",
       " 'dt16_ac140_ac140079.pdf',\n",
       " 'dt16_ac140_ac140086.pdf',\n",
       " 'dt16_ac140_ac140250.pdf',\n",
       " 'dt16_ac140_ac140267.pdf',\n",
       " 'dt16_ac141_ac141006.pdf',\n",
       " 'dt16_ac141_ac141013.pdf',\n",
       " 'dt16_ac141_ac141020.pdf',\n",
       " 'dt16_ac141_ac141201.pdf',\n",
       " 'dt16_ac141_ac141218.pdf',\n",
       " 'dt16_ac142_ac142143.pdf',\n",
       " 'dt16_ac142_ac142150.pdf',\n",
       " 'dt16_ac142_ac142167.pdf',\n",
       " 'dt16_ac143_ac143101.pdf',\n",
       " 'dt16_ac144_ac144036.pdf',\n",
       " 'dt16_ac144_ac144043.pdf',\n",
       " 'dt16_ac144_ac144050.pdf',\n",
       " 'dt16_ac144_ac144231.pdf',\n",
       " 'dt16_ac144_ac144248.pdf',\n",
       " 'dt16_ac145_ac145180.pdf',\n",
       " 'dt16_ac145_ac145197.pdf',\n",
       " 'dt16_ac146_ac146124.pdf',\n",
       " 'dt16_ac146_ac146131.pdf',\n",
       " 'dt17_ac147_ac147150.pdf',\n",
       " 'dt17_ac147_ac147167.pdf',\n",
       " 'dt17_ac147_ac147174.pdf',\n",
       " 'dt17_ac148_ac148101.pdf',\n",
       " 'dt17_ac148_ac148297.pdf',\n",
       " 'dt18_ac151_ac151103.pdf',\n",
       " 'dt18_ac151_ac151110.pdf',\n",
       " 'dt18_ac151_ac151127.pdf',\n",
       " 'dt18_ac152_ac152052.pdf',\n",
       " 'dt18_ac152_ac152069.pdf',\n",
       " 'dt18_ac152_ac152076.pdf',\n",
       " 'dt18_ac152_ac152240.pdf',\n",
       " 'dt18_ac152_ac152257.pdf',\n",
       " 'dt18_ac153_ac153003.pdf',\n",
       " 'dt18_ac153_ac153010.pdf',\n",
       " 'dt18_ac153_ac153199.pdf',\n",
       " 'dt18_ac154_ac154133.pdf',\n",
       " 'dt18_ac154_ac154140.pdf',\n",
       " 'dt18_ac155_ac155099.pdf',\n",
       " 'dt18_ac156_ac156026.pdf',\n",
       " 'dt18_ac156_ac156033.pdf',\n",
       " 'dt18_ac156_ac156214.pdf',\n",
       " 'dt18_ac156_ac156221.pdf',\n",
       " 'dt18_ac156_ac156238.pdf',\n",
       " 'dt18_ac157_ac157163.pdf',\n",
       " 'dt18_ac157_ac157170.pdf',\n",
       " 'dt18_ac157_ac157187.pdf',\n",
       " 'dt18_ac158_ac158107.pdf',\n",
       " 'dt18_ac158_ac158114.pdf',\n",
       " 'dt18_ac158_ac158121.pdf',\n",
       " 'dt19_ac160_ac160172.pdf',\n",
       " 'dt19_ac160_ac160189.pdf',\n",
       " 'dt19_ac160_ac160196.pdf',\n",
       " 'dt19_ac161_ac161116.pdf',\n",
       " 'dt19_ac161_ac161123.pdf',\n",
       " 'dt19_ac161_ac161130.pdf',\n",
       " 'dt19_ac162_ac162065.pdf',\n",
       " 'dt19_ac162_ac162072.pdf',\n",
       " 'dt19_ac162_ac162089.pdf',\n",
       " 'dt19_ac162_ac162260.pdf',\n",
       " 'dt19_ac163_ac163009.pdf',\n",
       " 'dt19_ac163_ac163016.pdf',\n",
       " 'dt19_ac163_ac163023.pdf',\n",
       " 'dt19_ac163_ac163204.pdf',\n",
       " 'dt19_ac163_ac163211.pdf',\n",
       " 'dt19_ac164_ac164153.pdf',\n",
       " 'dt19_ac164_ac164160.pdf',\n",
       " 'dt19_ac164_ac164177.pdf',\n",
       " 'dt20_ac166_ac166106.pdf',\n",
       " 'dt20_ac166_ac166113.pdf',\n",
       " 'dt20_ac166_ac166120.pdf',\n",
       " 'dt20_ac167_ac167055.pdf',\n",
       " 'dt20_ac167_ac167062.pdf',\n",
       " 'dt20_ac167_ac167079.pdf',\n",
       " 'dt20_ac167_ac167250.pdf',\n",
       " 'dt20_ac167_ac167267.pdf',\n",
       " 'dt20_ac168_ac168006.pdf',\n",
       " 'dt20_ac168_ac168013.pdf',\n",
       " 'dt20_ac168_ac168201.pdf',\n",
       " 'dt20_ac169_ac169143.pdf',\n",
       " 'dt20_ac169_ac169150.pdf',\n",
       " 'dt21_ac170_ac170064.pdf',\n",
       " 'dt21_ac170_ac170071.pdf',\n",
       " 'dt21_ac170_ac170088.pdf',\n",
       " 'dt21_ac170_ac170269.pdf',\n",
       " 'dt21_ac170_ac170276.pdf',\n",
       " 'dt21_ac171_ac171008.pdf',\n",
       " 'dt21_ac171_ac171015.pdf',\n",
       " 'dt21_ac171_ac171022.pdf',\n",
       " 'dt21_ac171_ac171203.pdf',\n",
       " 'dt21_ac171_ac171210.pdf',\n",
       " 'dt21_ac172_ac172152.pdf',\n",
       " 'dt21_ac172_ac172169.pdf',\n",
       " 'dt21_ac173_ac173103.pdf',\n",
       " 'dt21_ac173_ac173110.pdf',\n",
       " 'dt21_ac173_ac173282.pdf',\n",
       " 'dt21_ac174_ac174045.pdf',\n",
       " 'dt21_ac174_ac174052.pdf',\n",
       " 'dt21_ac174_ac174226.pdf',\n",
       " 'dt21_ac174_ac174233.pdf',\n",
       " 'dt21_ac174_ac174240.pdf',\n",
       " 'dt21_ac175_ac175010.pdf',\n",
       " 'dt21_ac175_ac175175.pdf',\n",
       " 'dt21_ac175_ac175182.pdf',\n",
       " 'dt21_ac175_ac175199.pdf',\n",
       " 'dt21_ac176_ac176119.pdf',\n",
       " 'dt21_ac176_ac176126.pdf',\n",
       " 'dt21_ac176_ac176133.pdf',\n",
       " 'dt21_ac177_ac177068.pdf',\n",
       " 'dt21_ac177_ac177075.pdf',\n",
       " 'dt21_ac177_ac177082.pdf',\n",
       " 'dt23_ac184_ac184135.pdf',\n",
       " 'dt23_ac184_ac184142.pdf',\n",
       " 'dt23_ac184_ac184323.pdf',\n",
       " 'dt23_ac184_ac184330.pdf',\n",
       " 'dt23_ac185_ac185084.pdf',\n",
       " 'dt23_ac185_ac185091.pdf',\n",
       " 'dt23_ac185_ac185272.pdf',\n",
       " 'dt23_ac185_ac185289.pdf',\n",
       " 'dt23_ac186_ac186028.pdf',\n",
       " 'dt23_ac186_ac186035.pdf',\n",
       " 'dt23_ac186_ac186042.pdf',\n",
       " 'dt23_ac186_ac186216.pdf',\n",
       " 'dt23_ac186_ac186223.pdf',\n",
       " 'dt23_ac187_ac187165.pdf',\n",
       " 'dt23_ac187_ac187172.pdf',\n",
       " 'dt25_ac198_ac198001.pdf',\n",
       " 'dt25_ac198_ac198018.pdf',\n",
       " 'dt25_ac198_ac198025.pdf',\n",
       " 'dt25_ac198_ac198206.pdf',\n",
       " 'dt25_ac198_ac198213.pdf',\n",
       " 'dt25_ac199_ac199148.pdf',\n",
       " 'dt25_ac199_ac199155.pdf',\n",
       " 'dt25_ac199_ac199162.pdf',\n",
       " 'dt25_ac200_ac200077.pdf',\n",
       " 'dt25_ac200_ac200084.pdf',\n",
       " 'dt25_ac200_ac200091.pdf',\n",
       " 'dt25_ac200_ac200265.pdf',\n",
       " 'dt25_ac200_ac200272.pdf',\n",
       " 'dt25_ac201_ac201011.pdf',\n",
       " 'dt25_ac201_ac201028.pdf',\n",
       " 'dt25_ac201_ac201209.pdf',\n",
       " 'dt25_ac201_ac201216.pdf',\n",
       " 'dt26_ac202_ac202047.pdf',\n",
       " 'dt26_ac202_ac202054.pdf',\n",
       " 'dt26_ac202_ac202061.pdf',\n",
       " 'dt26_ac202_ac202235.pdf',\n",
       " 'dt26_ac202_ac202242.pdf',\n",
       " 'dt26_ac203_ac203005.pdf',\n",
       " 'dt26_ac203_ac203184.pdf',\n",
       " 'dt26_ac203_ac203191.pdf',\n",
       " 'dt26_ac204_ac204128.pdf',\n",
       " 'dt26_ac204_ac204135.pdf',\n",
       " 'dt26_ac204_ac204142.pdf',\n",
       " 'dt26_ac205_ac205077.pdf',\n",
       " 'dt26_ac205_ac205084.pdf',\n",
       " 'dt26_ac205_ac205091.pdf',\n",
       " 'dt26_ac205_ac205265.pdf',\n",
       " 'dt26_ac205_ac205272.pdf',\n",
       " 'dt26_ac206_ac206011.pdf',\n",
       " 'dt26_ac206_ac206028.pdf',\n",
       " 'dt26_ac206_ac206209.pdf',\n",
       " 'dt26_ac206_ac206216.pdf',\n",
       " 'dt26_ac206_ac206223.pdf',\n",
       " 'dt26_ac207_ac207158.pdf',\n",
       " 'dt26_ac207_ac207165.pdf',\n",
       " 'dt26_ac207_ac207172.pdf',\n",
       " 'dt26_ac208_ac208109.pdf',\n",
       " 'dt26_ac208_ac208116.pdf',\n",
       " 'dt27_ac209_ac209128.pdf',\n",
       " 'dt27_ac209_ac209135.pdf',\n",
       " 'dt27_ac209_ac209142.pdf',\n",
       " 'dt27_ac210_ac210167.pdf',\n",
       " 'dt27_ac210_ac210174.pdf',\n",
       " 'dt27_ac210_ac210181.pdf',\n",
       " 'dt27_ac211_ac211101.pdf',\n",
       " 'dt27_ac211_ac211118.pdf',\n",
       " 'dt27_ac211_ac211125.pdf',\n",
       " 'dt27_ac211_ac211306.pdf',\n",
       " 'dt27_ac211_ac211313.pdf',\n",
       " 'dt27_ac212_ac212050.pdf',\n",
       " 'dt27_ac212_ac212067.pdf',\n",
       " 'dt27_ac212_ac212074.pdf',\n",
       " 'dt27_ac212_ac212248.pdf',\n",
       " 'dt27_ac212_ac212255.pdf',\n",
       " 'dt27_ac212_ac212262.pdf',\n",
       " 'dt28_ac213_ac213086.pdf',\n",
       " 'dt28_ac213_ac213093.pdf',\n",
       " 'dt28_ac214_ac214037.pdf',\n",
       " 'dt28_ac214_ac214044.pdf',\n",
       " 'dt28_ac214_ac214218.pdf',\n",
       " 'dt28_ac214_ac214225.pdf',\n",
       " 'dt28_ac214_ac214232.pdf',\n",
       " 'dt28_ac215_ac215167.pdf',\n",
       " 'dt28_ac215_ac215174.pdf',\n",
       " 'dt28_ac215_ac215181.pdf',\n",
       " 'dt28_ac216_ac216101.pdf',\n",
       " 'dt28_ac216_ac216118.pdf',\n",
       " 'dt28_ac216_ac216125.pdf',\n",
       " 'dt28_ac217_ac217050.pdf',\n",
       " 'dt28_ac217_ac217067.pdf',\n",
       " 'dt28_ac217_ac217074.pdf',\n",
       " 'dt28_ac218_ac218001.pdf',\n",
       " 'dt28_ac218_ac218018.pdf',\n",
       " 'dt28_ac218_ac218206.pdf',\n",
       " 'dt28_ac218_ac218213.pdf',\n",
       " 'dt29_ac219_ac219020.pdf',\n",
       " 'dt29_ac219_ac219037.pdf',\n",
       " 'dt29_ac219_ac219044.pdf',\n",
       " 'dt29_ac219_ac219225.pdf',\n",
       " 'dt29_ac219_ac219232.pdf',\n",
       " 'dt29_ac219_ac219256.pdf',\n",
       " 'dt29_ac220_ac220069.pdf',\n",
       " 'dt29_ac220_ac220076.pdf',\n",
       " 'dt29_ac220_ac220083.pdf',\n",
       " 'dt29_ac220_ac220257.pdf',\n",
       " 'dt29_ac220_ac220264.pdf',\n",
       " 'dt29_ac220_ac220271.pdf',\n",
       " 'dt29_ac221_ac221003.pdf',\n",
       " 'dt29_ac221_ac221010.pdf',\n",
       " 'dt29_ac221_ac221027.pdf',\n",
       " 'dt29_ac221_ac221208.pdf',\n",
       " 'dt29_ac221_ac221215.pdf',\n",
       " 'dt29_ac222_ac222157.pdf',\n",
       " 'dt29_ac222_ac222164.pdf',\n",
       " 'dt29_ac223_ac223108.pdf',\n",
       " 'dt29_ac223_ac223287.pdf',\n",
       " 'dt29_ac223_ac223294.pdf',\n",
       " 'dt29_ac224_ac224040.pdf',\n",
       " 'dt29_ac224_ac224057.pdf',\n",
       " 'dt29_ac224_ac224064.pdf',\n",
       " 'dt29_ac224_ac224221.pdf',\n",
       " 'dt29_ac224_ac224238.pdf',\n",
       " 'dt29_ac224_ac224245.pdf',\n",
       " 'dt29_ac225_ac225187.pdf',\n",
       " 'dt29_ac225_ac225194.pdf',\n",
       " 'dt29_ac226_ac226114.pdf',\n",
       " 'dt29_ac226_ac226121.pdf',\n",
       " 'dt29_ac226_ac226138.pdf',\n",
       " 'dt29_ac227_ac227063.pdf',\n",
       " 'dt29_ac227_ac227070.pdf',\n",
       " 'dt29_ac227_ac227087.pdf',\n",
       " 'dt29_ac227_ac227268.pdf',\n",
       " 'dt29_ac227_ac227275.pdf',\n",
       " 'dt29_ac227_ac227282.pdf',\n",
       " 'dt29_ac228_ac228007.pdf',\n",
       " 'dt29_ac228_ac228014.pdf',\n",
       " 'dt29_ac228_ac228021.pdf',\n",
       " 'dt29_ac228_ac228202.pdf',\n",
       " 'dt29_ac228_ac228219.pdf',\n",
       " 'dt31_ac149_ac149030.pdf',\n",
       " 'dt31_ac149_ac149047.pdf',\n",
       " 'dt31_ac149_ac149054.pdf',\n",
       " 'dt31_ac149_ac149228.pdf',\n",
       " 'dt31_ac149_ac149235.pdf',\n",
       " 'dt31_ac150_ac150079.pdf',\n",
       " 'dt31_ac150_ac150086.pdf',\n",
       " 'dt31_ac150_ac150267.pdf',\n",
       " 'dt31_ac150_ac150274.pdf',\n",
       " 'dt31_ac150_ac150281.pdf',\n",
       " 'dt32_ac101_ac101077.pdf',\n",
       " 'dt32_ac101_ac101084.pdf',\n",
       " 'dt32_ac101_ac101091.pdf',\n",
       " 'dt32_ac101_ac101265.pdf',\n",
       " 'dt32_ac101_ac101272.pdf',\n",
       " 'dt32_ac102_ac102011.pdf',\n",
       " 'dt32_ac102_ac102028.pdf',\n",
       " 'dt32_ac102_ac102035.pdf',\n",
       " 'dt32_ac102_ac102209.pdf',\n",
       " 'dt32_ac102_ac102216.pdf',\n",
       " 'dt32_ac102_ac102223.pdf',\n",
       " 'dt32_ac112_ac112135.pdf',\n",
       " 'dt32_ac112_ac112142.pdf',\n",
       " 'dt32_ac112_ac112159.pdf',\n",
       " 'dt32_ac113_ac113084.pdf',\n",
       " 'dt32_ac113_ac113091.pdf',\n",
       " 'dt32_ac113_ac113289.pdf',\n",
       " 'dt32_ac113_ac113296.pdf',\n",
       " 'dt32_ac114_ac114028.pdf',\n",
       " 'dt32_ac114_ac114035.pdf',\n",
       " 'dt32_ac114_ac114042.pdf',\n",
       " 'dt32_ac114_ac114223.pdf',\n",
       " 'dt32_ac115_ac115172.pdf',\n",
       " 'dt32_ac115_ac115189.pdf',\n",
       " 'dt32_ac115_ac115360.pdf',\n",
       " 'dt32_ac115_ac115377.pdf',\n",
       " 'dt32_ac125_ac125100.pdf',\n",
       " 'dt32_ac126_ac126042.pdf',\n",
       " 'dt32_ac126_ac126059.pdf',\n",
       " 'dt32_ac126_ac126230.pdf',\n",
       " 'dt32_ac126_ac126247.pdf']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sort_nicely(state_pdfs_files)\n",
    "state_pdfs_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_mandal(a,b,c,d,img,crop_path):\n",
    "            \n",
    "    crop_img = crop_section(a,b,c,d,img)\n",
    "\n",
    "    crop_det_path = crop_path+\"det.jpg\"\n",
    "    crop_img.save(crop_det_path)\n",
    "\n",
    "    text = (pytesseract.image_to_string(crop_det_path, config='--psm 6', lang='eng+tam'))\n",
    "    text = text.split('\\n')\n",
    "    text = [ i for i in text if i!='' and i!='\\x0c']\n",
    "\n",
    "    return text\n",
    "    \n",
    "def get_stats(a,b,c,d,img,crop_path):\n",
    "\n",
    "    crop_img = crop_section(a,b,c,d,img)\n",
    "\n",
    "    crop_path = input_images_blocks_path+\"page/\"\n",
    "    create_path(crop_path)\n",
    "\n",
    "    crop_stat_path = crop_path+\"stat.jpg\"\n",
    "    crop_img.save(crop_stat_path)\n",
    "\n",
    "    a_n,b_n,c_n,d_n = extract_4_numbers(crop_stat_path)\n",
    "\n",
    "    return a_n,b_n,c_n,d_n\n",
    "\n",
    "def get_police_det(a,b,c,d,img,crop_path):\n",
    "\n",
    "    crop_img = crop_section(a,b,c,d,img)\n",
    "\n",
    "    crop_police_path = crop_path+\"police.jpg\"\n",
    "    crop_img.save(crop_police_path)\n",
    "\n",
    "    text = (pytesseract.image_to_string(crop_police_path, config='--psm 6', lang='eng+tam')) #config='--psm 4' config='-c preserve_interword_spaces=1'\n",
    "    text = text.split('\\n')\n",
    "    text = [ i for i in text if i!='' and i!='\\x0c']\n",
    "\n",
    "    return text\n",
    "\n",
    "def get_stats(a,b,c,d,img,crop_path):\n",
    "\n",
    "    crop_img = crop_section(a,b,c,d,img)\n",
    "\n",
    "    crop_path = input_images_blocks_path+\"page/\"\n",
    "    create_path(crop_path)\n",
    "\n",
    "    crop_stat_path = crop_path+\"stat.jpg\"\n",
    "    crop_img.save(crop_stat_path)\n",
    "\n",
    "    a_n,b_n,c_n,d_n = extract_4_numbers(crop_stat_path)\n",
    "\n",
    "    return a_n,b_n,c_n,d_n\n",
    "\n",
    "def extract_4_numbers(crop_stat_path):\n",
    "    \n",
    "    text = (pytesseract.image_to_string(crop_stat_path, config='--psm 11', lang='eng')) #config='--psm 4' config='-c preserve_interword_spaces=1'\n",
    "#     print(text)\n",
    "    text = re.findall(r'\\d+', text)\n",
    "#     print(text)\n",
    "\n",
    "    if len(text)==4:\n",
    "        if int(text[0]) + int(text[1]) == int(text[2]):\n",
    "            net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = text[0],text[1],\"0\",text[2]\n",
    "        elif int(text[0]) + int(text[1]) == int(text[3]):\n",
    "            net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = text[0],text[1],\"0\",text[3]\n",
    "        else:\n",
    "            net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = text[0],text[1],text[2],text[3]\n",
    "    elif len(text) == 3 and int(text[2])>=int(text[1]) and int(text[2])>=int(text[0]):\n",
    "        net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = text[0],text[1],\"0\",text[2]\n",
    "    elif len(text) == 2 and int(text[0])*2-100<int(text[1]):\n",
    "        net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = text[0],int(text[1])-int(text[0]),\"0\",text[1]\n",
    "    else:\n",
    "        net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = \"\",\"\",\"\",\"\"\n",
    "    \n",
    "    return net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_data(data):\n",
    "    seps = [\":\",\">\",\"-\",\".\"]\n",
    "    \n",
    "    for s in seps:\n",
    "        if s in data:\n",
    "            break\n",
    "\n",
    "    data = data.split(s)\n",
    "    data = [ i for i in data if i.strip()!='']\n",
    "    if len(data)>1:\n",
    "        data = data[1].strip()\n",
    "        return data\n",
    "    else:\n",
    "        data = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_name(name):\n",
    "    \n",
    "    row = name.split(\":\")\n",
    "    if len(row)!=2:\n",
    "        return \"\"\n",
    "    else:\n",
    "        return row[1].strip()\n",
    "    \n",
    "def extract_vid(v_id):\n",
    "    row = v_id.split(\"|\")\n",
    "    if len(row)>=2:\n",
    "        number = re.findall(r'\\d+', row[0].strip())\n",
    "        if len(number)>0:\n",
    "            return number[0],row[1]\n",
    "        else:\n",
    "            return \"\",row[1]\n",
    "        \n",
    "    row = v_id.split(\" \")\n",
    "    if len(row)>=2:\n",
    "        number = re.findall(r'\\d+', row[0].strip())\n",
    "        if len(number)>0:\n",
    "            return number[0],row[1].strip()\n",
    "        else:\n",
    "            return \"\",row[1].strip()\n",
    "    \n",
    "    else:\n",
    "        return \"\",v_id.strip()\n",
    "\n",
    "def extract_house_no(house_no):\n",
    "    row = house_no.split(\":\")\n",
    "    if len(row)==2:\n",
    "        house_no = row[1].strip()\n",
    "        return house_no\n",
    "    else:\n",
    "        house_no = re.findall(r'\\d+', row[0].strip())\n",
    "        if len(house_no)>0:\n",
    "            return house_no[0]\n",
    "        else:\n",
    "            return \"\"\n",
    "    \n",
    "def extract_age_gender(age_gender):\n",
    "    \n",
    "    text = re.findall(r'\\d+', age_gender)\n",
    "    \n",
    "    if len(text)>0:\n",
    "        age = text[0]\n",
    "    else:\n",
    "        age = \"\"\n",
    "        \n",
    "    if \"ஆண்\" in age_gender:\n",
    "        gender = \"Male\"\n",
    "    elif \"பெண்\" in age_gender:\n",
    "        gender =  \"Female\" \n",
    "    else:\n",
    "        gender = 'Male'\n",
    "        \n",
    "    return age, gender\n",
    "\n",
    "def extract_rel_name(rel_name):\n",
    "    row = rel_name.split(\":\")\n",
    "    if len(row)==2:\n",
    "        \n",
    "        rel_type = extract_rel_type(row[0].strip())\n",
    "        \n",
    "        return row[1].strip(),rel_type\n",
    "        \n",
    "    else:\n",
    "        row = rel_name.split(\";\")\n",
    "        if len(row)!=2:\n",
    "            return \"\",\"\"\n",
    "        else:\n",
    "            rel_type = extract_rel_type(row[0].strip())\n",
    "            return row[1].strip(),rel_type\n",
    "        \n",
    "    \n",
    "def extract_rel_type(rel_type):\n",
    "    \n",
    "    if \"கணவர்\" in rel_type:\n",
    "        rel_type = 'husband'\n",
    "    elif 'தந்' in rel_type :\n",
    "        rel_type = 'father'\n",
    "    else:\n",
    "        rel_type = 'father'\n",
    "    \n",
    "    return rel_type \n",
    "\n",
    "\n",
    "def extract_details_from_block(block):\n",
    "    \n",
    "    if len(block) == 5:\n",
    "        v_id = block[0]\n",
    "        name = block[1]\n",
    "        rel_name = block[2]\n",
    "        house_no = block[3]\n",
    "        age_gender = block[4]\n",
    "    else:\n",
    "        v_id = block[0]\n",
    "        name = block[1]+\" \"+ block[2]\n",
    "        rel_name = block[3]\n",
    "        house_no = block[4]\n",
    "        age_gender = block[5]\n",
    "    \n",
    "    name = extract_name(name)\n",
    "    rel_name,rel_type = extract_rel_name(rel_name)\n",
    "    house_no = extract_house_no(house_no)\n",
    "    age, gender = extract_age_gender(age_gender)\n",
    "    number,voter_id = extract_vid(v_id)\n",
    "    \n",
    "    \n",
    "    return [name,rel_name,rel_type,house_no,age,gender,voter_id,number]\n",
    "\n",
    "def arrange_columns(first_page_list,block_list,filename,a_n,b_n,c_n,d_n):\n",
    "    \n",
    "    year = 2018\n",
    "    state = 'tamilnadu'\n",
    "    \n",
    "    ac_name,parl_constituency,part_no,main_town,police_station,polling_station_name,polling_station_address,revenue_division,mandal,district,pin_code,net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = first_page_list\n",
    "    name,rel_name,rel_type,house_no,age,gender,voter_id,number = block_list\n",
    "    \n",
    "    if net_electors_male ==\"\" and net_electors_female==\"\":\n",
    "        net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = a_n,b_n,c_n,d_n\n",
    "    \n",
    "    final_list = [number,voter_id,name,rel_name,rel_type,house_no,age,gender,ac_name,\n",
    "                 parl_constituency,part_no,year,state,filename,main_town,police_station,mandal,\n",
    "                 revenue_division,district,pin_code,polling_station_name,polling_station_address,\n",
    "                 net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total]\n",
    "\n",
    "\n",
    "    return final_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_poll_blocks_from_page(page_full_path,page_blocks_path,amend_page, height):\n",
    "    \n",
    "    img = Image.open(page_full_path)\n",
    "    \n",
    "    def generate(intial_width,a,b,gap):\n",
    "        count = 0\n",
    "        crop_width = 1245\n",
    "        crop_height = 481\n",
    "\n",
    "        for col in range(1,11):\n",
    "\n",
    "            for row in range(1,4):\n",
    "                c = a+crop_width\n",
    "                d = b+crop_height\n",
    "                area = (a, b, c, d)\n",
    "                cropped_img = img.crop(area)\n",
    "                count = count+1\n",
    "                cropped_img.save(page_blocks_path+str(count)+\".jpg\")\n",
    "\n",
    "                a = c\n",
    "\n",
    "            a = intial_width\n",
    "            b = b+crop_height+gap\n",
    "    \n",
    "    intial_width = 290\n",
    "    generate(intial_width,intial_width,height,0)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_first_page_details(path):\n",
    "    \n",
    "    img = Image.open(path)\n",
    "    \n",
    "    crop_path = input_images_blocks_path+\"page/\"\n",
    "    create_path(crop_path)\n",
    "    \n",
    "    a,b,c,d = 2156,4444,1490,148  # stats for male and female\n",
    "\n",
    "    a_n,b_n,c_n,d_n = \"\",\"\",\"\",\"\"\n",
    "    a_n,b_n,c_n,d_n = get_stats(a,b,c,d,img,crop_path)\n",
    "        \n",
    "    a,b,c,d = 1062,3322,1200,644 # police name name and address\n",
    "    \n",
    "    text = get_police_det(a,b,c,d,img,crop_path)\n",
    "    \n",
    "    polling_station_name, polling_station_address = \"\",\"\"\n",
    "\n",
    "    \n",
    "    if len(text) >=5:\n",
    "        pass\n",
    "    \n",
    "    elif len(text)>1:\n",
    "        idx = int(len(text)/2)\n",
    "    \n",
    "        psn,psa = text[:idx], text[idx:]\n",
    "\n",
    "        for p in psn:\n",
    "            polling_station_name = polling_station_name +\" \" +p\n",
    "        for p in psa:\n",
    "            polling_station_address = polling_station_address +\" \" +p\n",
    "            \n",
    "        polling_station_name = polling_station_name.strip()\n",
    "        polling_station_address = polling_station_address.strip()\n",
    "    \n",
    "    a,b,c,d = 3073,2180,701,960  # mandal block\n",
    "    main_town,police_station,revenue_division,mandal,district,pin_code = \"\",\"\",\"\",\"\",\"\",\"\"\n",
    "\n",
    "    text = get_mandal(a,b,c,d,img,crop_path)\n",
    "    \n",
    "    if len(text) == 7:\n",
    "\n",
    "        main_town = text[0]\n",
    "        police_station = text[3]\n",
    "        revenue_division = text[1]\n",
    "        mandal = text[4]\n",
    "        district = text[5]\n",
    "        pin_code = text[6]\n",
    "\n",
    "    elif len(text) == 5:\n",
    "\n",
    "        main_town = \"\"\n",
    "        police_station = text[0]\n",
    "        revenue_division = text[1]\n",
    "        mandal = text[2]\n",
    "        district = text[3]\n",
    "        pin_code = text[4]\n",
    "    \n",
    "    elif len(text) == 6:\n",
    "\n",
    "        main_town = text[0]\n",
    "        police_station = text[2]\n",
    "        revenue_division = text[1]\n",
    "        mandal = text[3]\n",
    "        district = text[4]\n",
    "        pin_code = text[5]\n",
    "    \n",
    "    elif len(text) == 4:\n",
    "\n",
    "        main_town = text[0]\n",
    "        police_station = \"\"\n",
    "        revenue_division = \"\"\n",
    "        mandal = text[1]\n",
    "        district = text[2]\n",
    "        pin_code = text[3]\n",
    "\n",
    "    \n",
    "    a,b,c,d = 3378,422,380,200 # part no\n",
    "    crop_img = crop_section(a,b,c,d,img)\n",
    "    \n",
    "    crop_part_path = crop_path+\"part.jpg\"\n",
    "    crop_img.save(crop_part_path)\n",
    "\n",
    "    text = (pytesseract.image_to_string(crop_part_path, config='--psm 6', lang='eng+tam')) #config='--psm 4' config='-c preserve_interword_spaces=1'\n",
    "    text = re.findall(r'\\d+', text)\n",
    "            \n",
    "    \n",
    "    if len(text)>0:\n",
    "        part_no = text[0]\n",
    "    else:\n",
    "        part_no = \"\"\n",
    "        \n",
    "    \n",
    "    a,b,c,d = 352,417,2700,542 # ac name and parl\n",
    "    crop_img = crop_section(a,b,c,d,img)\n",
    "    \n",
    "    crop_ac_path = crop_path+\"ac.jpg\"\n",
    "    crop_img.save(crop_ac_path)\n",
    "    \n",
    "    text = (pytesseract.image_to_string(crop_ac_path, config='--psm 6', lang='tam')) #config='--psm 4' config='-c preserve_interword_spaces=1'\n",
    "    text = text.split('\\n')\n",
    "    text = [ i for i in text if i!='' and i!='\\x0c']\n",
    "    \n",
    "    ac_name,parl_constituency = '',''\n",
    "    \n",
    "    if len(text) >= 5:\n",
    "        \n",
    "        an = text[0].split('மற்றும்')\n",
    "        if len(an)>0:\n",
    "            ac_name = an[-1]\n",
    "        \n",
    "        pc = text[2].split('அடங்கியுள்ள')\n",
    "        \n",
    "        if len(pc)>1:\n",
    "            parl_constituency = pc[1]\n",
    "        else:\n",
    "            pc = text[2].split('வணகயில்')\n",
    "        \n",
    "            if len(pc)>1:\n",
    "                parl_constituency = pc[1]\n",
    "            else:\n",
    "                pc = text[2].split(\"பங்கய ப\")\n",
    "                if len(pc)>0:\n",
    "                    parl_constituency = pc[-1]\n",
    "        \n",
    "        ac_name = ac_name.strip()\n",
    "        parl_constituency = parl_constituency.strip()\n",
    "        \n",
    "   \n",
    "    return [ac_name,parl_constituency,part_no,main_town,police_station,polling_station_name,polling_station_address,revenue_division,mandal,district,pin_code,a_n,b_n,c_n,d_n]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dt3_ac036_ac036205.pdf\n",
      "page done :  3.jpg\n",
      "page done :  4.jpg\n",
      "page done :  5.jpg\n",
      "page done :  6.jpg\n",
      "page done :  7.jpg\n",
      "page done :  8.jpg\n",
      "page done :  9.jpg\n",
      "page done :  10.jpg\n",
      "page done :  11.jpg\n",
      "page done :  12.jpg\n",
      "page done :  13.jpg\n",
      "page done :  14.jpg\n",
      "page done :  15.jpg\n",
      "page done :  16.jpg\n",
      "page done :  17.jpg\n",
      "page done :  18.jpg\n",
      "CSV saved\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for pdf_file_name in ['dt3_ac036_ac036205.pdf']:\n",
    "# for pdf_file_name in state_pdfs_files:\n",
    "    \n",
    "    print(pdf_file_name)\n",
    "    \n",
    "    #create images,blocks and csvs paths for each file\n",
    "    pdf_file_name_without_ext = pdf_file_name.split('.pdf')[0]\n",
    "    input_pdf_images_path = PARSE_DATA_PAGES+pdf_file_name_without_ext+\"/\"\n",
    "    create_path(input_pdf_images_path)\n",
    "\n",
    "    input_images_blocks_path = PARSE_DATA_BLOCKS+pdf_file_name_without_ext+\"/\"\n",
    "    create_path(input_images_blocks_path)\n",
    "    \n",
    "    if os.path.exists(PARSE_DATA_CSVS+pdf_file_name_without_ext+\".csv\"):\n",
    "        print(pdf_file_name_without_ext+\".csv\", \"already exists\")\n",
    "        continue\n",
    "            \n",
    "    #convert pdf into bunch of images\n",
    "    try:\n",
    "        pdf_2_images_list = pdf_to_img(state_pdfs_path+pdf_file_name, input_pdf_images_path,dpi=500)\n",
    "    except:\n",
    "        continue\n",
    "    \n",
    "    #sort pages for looping\n",
    "    input_images = os.listdir(input_pdf_images_path)\n",
    "    sort_nicely(input_images)\n",
    "    \n",
    "    #empty intial data\n",
    "    df = pd.DataFrame(columns = COLUMNS)\n",
    "    order_problem = []\n",
    "    \n",
    "    amend_page = False\n",
    "    a_n,b_n,c_n,d_n = \"\",\"\",\"\",\"\"\n",
    "    \n",
    "    #for each page, parse the data\n",
    "    for page in input_images:\n",
    "    \n",
    "        page_full_path = input_pdf_images_path+page\n",
    "        \n",
    "        #extract first page content\n",
    "        if page == '1.jpg':\n",
    "            first_page_list = extract_first_page_details(page_full_path)\n",
    "            continue \n",
    "            \n",
    "\n",
    "        #ingnore 2nd page and last page\n",
    "        if page == '2.jpg':\n",
    "            continue\n",
    "            \n",
    "        if input_images[-1] == page:\n",
    "            page_full_path = input_pdf_images_path+page\n",
    "            img = Image.open(page_full_path)\n",
    "            \n",
    "            crop_path = input_images_blocks_path+\"page/\"\n",
    "            create_path(crop_path)\n",
    "                        \n",
    "            a,b,c,d = 2843,1377,950,183  # stats for male and female\n",
    "            a_n,b_n,c_n,d_n = get_stats(a,b,c,d,img,crop_path)\n",
    "            break\n",
    "        \n",
    "        #loop from 3 page onwards\n",
    "        if page.endswith('.jpg'):\n",
    "            \n",
    "            final_invidual_blocks = []\n",
    "            blocks_path = input_images_blocks_path+\"blocks/\"\n",
    "            create_path(blocks_path)\n",
    "\n",
    "            page_idx = page.split(\".jpg\")[0] + \"/\"\n",
    "            page_blocks_path = blocks_path+page_idx\n",
    "            create_path(page_blocks_path)\n",
    "            \n",
    "            if page == '3.jpg':\n",
    "                generate_poll_blocks_from_page(page_full_path,page_blocks_path,amend_page, 484)\n",
    "            else:\n",
    "                \n",
    "                generate_poll_blocks_from_page(page_full_path,page_blocks_path,amend_page, 444)\n",
    "            \n",
    "            sorted_blocks = os.listdir(page_blocks_path)\n",
    "            sort_nicely(sorted_blocks)\n",
    "            \n",
    "            for jpg_file in sorted_blocks:\n",
    "                \n",
    "                if jpg_file.endswith('.jpg') :\n",
    "\n",
    "                    text = (pytesseract.image_to_string(page_blocks_path+jpg_file, config='--psm 6', lang='eng+tam')) #config='--psm 4' config='-c preserve_interword_spaces=1'\n",
    "                    params_list = text.split('\\n')\n",
    "                    new_params_list = [ i for i in params_list if i!='' and i!='\\x0c']\n",
    "                    \n",
    "                    if len(new_params_list) ==5 or len(new_params_list) == 6:\n",
    "                        final_invidual_blocks.append(new_params_list)\n",
    "                    else:\n",
    "                        order_problem.append((page, jpg_file,new_params_list))    \n",
    "\n",
    "        \n",
    "        #put the data into dataframe\n",
    "        for block in final_invidual_blocks:\n",
    "            block_list = extract_details_from_block(block)\n",
    "            \n",
    "            final_list = arrange_columns(first_page_list,block_list,pdf_file_name_without_ext,a_n,b_n,c_n,d_n)\n",
    "            \n",
    "            df_length = len(df)\n",
    "            df.loc[df_length] = final_list\n",
    "        \n",
    "        print(\"page done : \",page)\n",
    "                \n",
    "    #save the dataframe(pdf) data into csv\n",
    "    save_to_csv(df,PARSE_DATA_CSVS+pdf_file_name_without_ext+\".csv\")\n",
    "    print(\"CSV saved\")\n",
    "    print()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#combine all state files into one csv\n",
    "combine_all_csvs(\"tamilnadu_final.csv\",PARSE_DATA_CSVS)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}