.env

echo -e """\n\t\t To download PIB articles for a range of date do the following...\n
		\t\t pib startDate endDate (format: yyyy-mm-dd)\n
		\t\t for example to download articles from June 1st,2021 to Jun 15th,2021 enter\n
		\t\t pib 2021-06-01 2021-07-01 \n

		for Entire Month of 2021 Enter (for example month of January)
		\t\t pib_month Jan 2021

		for present day
		\t\t pib_today

		for last day
		\t\t pib_last_day

                for particular ministry and dates
                \t\t pib_min start_date end_date ministry_code

                to get ministry codes either check ministries.txt or run
                \t\t list_ministries
		"""




function is_leap(){

	yr=$1

	if [[ ( $(($yr%4)) == 0 ) && ( $(($yr%100)) != 0 || $(($yr%400)) == 0 ) ]]; then
		true;return
	else
		false;return
	fi
}


function pib_iter_dates(){

  if [ $# = 2 ] || [ $# = 3 ]; then
  	s_date=$(date -I --date="$1")
  	e_date=$(date -I --date="$2+1day")
  	while [ "${s_date}" != "${e_date}" ] ;do
  		echo -e "\nWorking on articles for the day\t:\t${s_date}"
  		
  		cyear=$(date "+%Y" --date=${s_date}) || echo -e "enter correct date in format YYYY-MM-DD"

  		if [ ${cyear} -ge 2017 ]; then
                sel_spider="pib"
        else
                sel_spider="pib_archives"
        fi
        
  		if [ $# -eq 2 ]; then
  		  scrapy crawl --nolog ${sel_spider} -a rel_date="${s_date}" -a rel_mincode="0"
  		fi
  		
  		if [ $# -eq 3 ]; then
  		  min_code=$3
  		  scrapy crawl --nolog ${sel_spider} -a rel_date="${s_date}" -a rel_mincode=${min_code}
  		fi
  		
  		s_date=$(date -I --date="$s_date+1day")
  	done
	else
	  echo -e "\nEnter correct set of arguments (date date ministry_code)\n Enter Date in format of YYYY-MM-DD"
	fi
}

function pib(){
  sdate=$1
  edate=$2
  pib_iter_dates ${sdate} ${edate}
}

function pib_dd_min(){
  sdate=$1
  edate=$2
  min_code=$3
  pib_iter_dates ${sdate} ${edate} ${min_code}
}

function pib_month(){
	mon=$1
	yr=$2
	shopt -s nocasematch
	case ${mon} in
		Jan)
			pib $yr-01-01 $yr-01-31
			;;
		Feb)
			if is_leap $yr; then
				echo -e "\n\n$yr is a Leap Year\n"
				pib $yr-02-01 $yr-02-29
			else
				pib $yr-02-01 $yr-02-28
			fi
			;;
		Mar)
			pib $yr-03-01 $yr-03-31
			;;
		Apr)
			pib $yr-04-01 $yr-04-30
			;;
		May)
			pib $yr-05-01 $yr-05-31
			;;
		Jun)
			pib $yr-06-01 $yr-06-30
			;;
		Jul)
			pib $yr-07-01 $yr-07-31
			;;
		Aug)
			pib $yr-08-01 $yr-08-31
			;;
		Sep)
			pib $yr-09-01 $yr-09-30
			;;
		Oct)
			pib $yr-10-01 $yr-10-31
			;;
		Nov)
			pib $yr-11-01 $yr-11-30
			;;
		Dec)
			pib $yr-12-01 $yr-12-31
			;;
		*)
			echo -e "\nPlease Enter a correct month (Format : mmm)\nExample: Jan for January\npib_month Jan 2021"
			;;
	esac
}

function pib_today(){

	t_date=$(date +'%Y-%m-%d')
	scrapy crawl --nolog pib_daily -a rel_date="$t_date"
}

function pib_last_day(){

	t_date=$(date +'%Y-%m-%d')
	l_date=$(date -I --date="$t_date-1day")
	scrapy crawl --nolog pib_daily -a rel_date="$l_date"
 }

function pib_min(){
  idate_str=$1
  edate_str=$2
  ministry=$3
  sdate=$(date -I --date="${idate_str}")
  edate=$(date -I --date="${edate_str}")
  if ! [[ ${ministry} =~ ^[0-9]+$ ]];
  then
      echo -e "\nEnter ministry code"
      exit 1
  fi
  echo -e "\nStarting date\t:\t${sdate}"
  echo -e "\nEnd date\t:\t${edate}"
  echo -e "\nSelected ministry is\t:\t"
case $ministry in
    0)
        echo -e "\nAll Ministry"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    1)
        echo -e "\nPresident's Secretariat"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    2)
        echo -e "\nVice President's Secretariat"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    3)
        echo -e "\nPrime Minister's Office"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    61)
        echo -e "\nCabinet"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    62)
        echo -e "\nCabinet Committee Decisions"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    63)
        echo -e "\nCabinet Committee on Economic Affairs (CCEA)"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    68)
        echo -e "\nCabinet Secretariat"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    70)
        echo -e "\nCabinet Committee on Infrastructure"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    71)
        echo -e "\nCabinet Committee on Price"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    75)
        echo -e "\nCabinet Committee on Investment"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    80)
        echo -e "\nAYUSH"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    72)
        echo -e "\nOther Cabinet Committees"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    14)
        echo -e "\nDepartment of Space"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    45)
        echo -e "\nDepartment of Ocean Development"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    56)
        echo -e "\nDepartment of Atomic Energy"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    35)
        echo -e "\nElection Commission"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    1330)
        echo -e "\nFinance Commission"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    27)
        echo -e "\nMinistry of Agriculture & Farmers Welfare"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    58)
        echo -e "\nMinistry of Agro & Rural Industries"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    41)
        echo -e "\nMinistry of Chemicals and Fertilizers"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    26)
        echo -e "\nMinistry of Civil Aviation"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    42)
        echo -e "\nMinistry of Coa"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    16)
        echo -e "\nMinistry of Commerce & Industry"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    24)
        echo -e "\nMinistry of Communications"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    60)
        echo -e "\nMinistry of Company Affairs"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    39)
        echo -e "\nMinistry of Consumer Affairs, Food & Public Distribution"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    1440)
        echo -e "\nMinistry of Cooperation"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    66)
        echo -e "\nMinistry of Corporate Affairs"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    17)
        echo -e "\nMinistry of Culture"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    33)
        echo -e "\nMinistry of Defence"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    57)
        echo -e "\nMinistry of Development of North-East Region"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    48)
        echo -e "\nMinistry of Disinvestment"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    73)
        echo -e "\nMinistry of Drinking Water & Sanitation"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    67)
        echo -e "\nMinistry of Earth Sciences"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    8)
        echo -e "\nMinistry of Education"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    1323)
        echo -e "\nMinistry of Electronics & IT"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    30)
        echo -e "\nMinistry of Environment, Forest and Climate Change"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    4)
        echo -e "\nMinistry of External Affairs"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    15)
        echo -e "\nMinistry of Finance"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    1340)
        echo -e "\nMinistry of Fisheries, Animal Husbandry & Dairying"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    40)
        echo -e "\nMinistry of Food Processing Industries"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    31)
        echo -e "\nMinistry of Health and Family Welfare"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    53)
        echo -e "\nMinistry of Heavy Industries"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    5)
        echo -e "\nMinistry of Home Affairs"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    47)
        echo -e "\nMinistry of Housing & Urban Affairs"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    11)
        echo -e "\nMinistry of Information & Broadcasting"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    1336)
        echo -e "\nMinistry of Jal Shakti"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    21)
        echo -e "\nMinistry of Labour & Employment"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    7)
        echo -e "\nMinistry of Law and Justice"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    51)
        echo -e "\nMinistry of Micro, Small & Medium Enterprises"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    44)
        echo -e "\nMinistry of Mines"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    65)
        echo -e "\nMinistry of Minority Affairs"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    28)
        echo -e "\nMinistry of New and Renewable Energy"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    59)
        echo -e "\nMinistry of Overseas Indian Affairs"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    10)
        echo -e "\nMinistry of Panchayati Raj"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    12)
        echo -e "\nMinistry of Parliamentary Affairs"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    6)
        echo -e "\nMinistry of Personnel, Public Grievances & Pensions"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    20)
        echo -e "\nMinistry of Petroleum & Natural Gas"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    79)
        echo -e "\nMinistry of Planning"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    52)
        echo -e "\nMinistry of Power"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    23)
        echo -e "\nMinistry of Railways"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    69)
        echo -e "\nMinistry of Road Transport & Highways"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    43)
        echo -e "\nMinistry of Rural Development"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    13)
        echo -e "\nMinistry of Science & Technology"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    46)
        echo -e "\nMinistry of Ports, Shipping and Waterways"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    77)
        echo -e "\nMinistry of Skill Development and Entrepreneurship"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    50)
        echo -e "\nMinistry of Social Justice & Empowerment"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    55)
        echo -e "\nMinistry of Statistics & Programme Implementation"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    18)
        echo -e "\nMinistry of Steel"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    25)
        echo -e "\nMinistry of Surface Transport"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    19)
        echo -e "\nMinistry of Textiles"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    36)
        echo -e "\nMinistry of Tourism"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    49)
        echo -e "\nMinistry of Tribal Affairs"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    32)
        echo -e "\nMinistry of Urban Development"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    38)
        echo -e "\nMinistry of Water Resources, River Development and Ganga Rejuvenation"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    64)
        echo -e "\nMinistry of Women and Child Development"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    9)
        echo -e "\nMinistry of Youth Affairs and Sports"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    78)
        echo -e "\nNITI Aayog"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    1325)
        echo -e "\nPM Speech"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    74)
        echo -e "\nEAC-PM"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    34)
        echo -e "\nUPSC"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    37)
        echo -e "\nSpecial Service and Features"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    1005)
        echo -e "\nPIB Headquarters"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    1406)
        echo -e "\nOffice of Principal Scientific Advisor to GoI"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    1454)
        echo -e "\nNational Financial Reporting Authority"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    1458)
        echo -e "\nCompetition Commission of India"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    1470)
        echo -e "\nIFSC Authority"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    1484)
        echo -e "\nNational Security Council Secretariat"
        pib_dd_min ${sdate} ${edate} ${ministry}
        ;;
    *)
        echo -e "\nInvalid Ministry Code"
        ;;
esac
    
}


function pdftxt(){

	home_dir="/home/runner"
	pib_dir="$home_dir/pib"
	pib_text_dir="$home_dir/pib_text"

	mkdir -p $pib_dir;
	mkdir -p $pib_text_dir;
	
	cd $pib_dir;
	
	for i in $(fd -e pdf); do
		echo $i;
		pdftotext $i;
	done

	for j in $(fd -e txt); do
		echo $j;
		cp -R -u --parents $j $pib_text_dir;
		rm -f $j;
	done
	cd $home_dir
}

function err_handle(){

	err_code=$1
	err_desc=$2
	echo -e "\nError ${err_code} : ${err_desc}"
	true
}

function list_ministries(){
  cat ministries.txt
}