GoIndex_Downloader.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "GoIndex Downloader.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMb5tv29S2FnYTw7FiTo+2y",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/anurag2947/GoIndex-Downloader/blob/master/GoIndex_Downloader.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7_jk3W0Zc5cy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive', force_remount=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kCkSzl3ANnMi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!apt update\n",
        "!pip install selenium\n",
        "!apt install chromium-chromedriver"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "APVi5oHSoy4r",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import shutil\n",
        "import urllib.parse\n",
        "import sys\n",
        "\n",
        "from time import sleep\n",
        "\n",
        "import requests\n",
        "from tqdm import tqdm\n",
        "\n",
        "sys.path.insert(0,'/usr/lib/chromium-browser/chromedriver')\n",
        "\n",
        "from selenium import webdriver\n",
        "\n",
        "from bs4 import BeautifulSoup\n",
        "from selenium.webdriver.common.by import By\n",
        "from selenium.webdriver.support.ui import WebDriverWait\n",
        "from selenium.webdriver.support import expected_conditions as EC\n",
        "\n",
        "\n",
        "def getLinks(pageUrl, path):\n",
        "    all_links_text = \"\"\n",
        "    dirs = \"\"\n",
        "    driver.get(pageUrl)\n",
        "    try:\n",
        "      WebDriverWait(driver, 30).until(EC.invisibility_of_element_located((By.CLASS_NAME, \"mdui-progress\")))\n",
        "    except:\n",
        "      print(\"Site took too long to load\")\n",
        "      sys.exit()\n",
        "    html = driver.execute_script(\"return document.getElementsByTagName('html')[0].innerHTML\")\n",
        "\n",
        "    page_source = BeautifulSoup(html, 'lxml')\n",
        "    tags_a_list = page_source.find('ul').find_all_next('a')\n",
        "\n",
        "\n",
        "    for link in tags_a_list:\n",
        "\n",
        "        currentLink = domain + str(link.get('href'))\n",
        "\n",
        "        # checking whether currentLink is a link to folder or file\n",
        "\n",
        "        # link to folder\n",
        "        if currentLink[-1] == '/':\n",
        "            sublink = currentLink[:currentLink.rindex('/')]\n",
        "            dirname = sublink[sublink.rindex('/') + 1:]\n",
        "\n",
        "            print(dirname)\n",
        "\n",
        "            dirs = dirs + dirname + '\\n'  # puts the name of currently made dir in dirs\n",
        "\n",
        "            if os.path.exists(path + dirname):\n",
        "                shutil.rmtree(path + dirname)\n",
        "            os.makedirs(path + dirname)\n",
        "\n",
        "            getLinks(currentLink, path + dirname + \"/\")\n",
        "\n",
        "        # Else it is a link to file\n",
        "        else:\n",
        "            all_links_text = all_links_text + currentLink.replace(\"?a=view\", \"\").replace(\" \", \"%20\") + \"\\n\"\n",
        "\n",
        "    # writing all links to links.txt\n",
        "    if all_links_text != \"\":\n",
        "        all_links_text = all_links_text[:all_links_text.rindex('\\n')]\n",
        "        fileLinks = open(path + \"links.txt\", 'w')\n",
        "        fileLinks.write(all_links_text)\n",
        "        fileLinks.close()\n",
        "\n",
        "    # writing all directories names to dirs.txt\n",
        "    if dirs != \"\":\n",
        "        dirs = dirs[:dirs.rindex('\\n')]\n",
        "        fileDirs = open(path + \"dirs.txt\", 'w')\n",
        "        fileDirs.write(dirs)\n",
        "        fileDirs.close()\n",
        "\n",
        "\n",
        "def file_downloader(download_url, file_store_path, file_name):\n",
        "    r = requests.get(download_url, stream=True)\n",
        "\n",
        "    try:\n",
        "        total_size = int(r.headers['content-length'])\n",
        "    except:\n",
        "        total_size = 1\n",
        "    with open(file_store_path + file_name, 'wb') as f:\n",
        "        for data in tqdm(iterable=r.iter_content(chunk_size=chunkSize), total=total_size / chunkSize, unit='KB'):\n",
        "            f.write(data)\n",
        "\n",
        "\n",
        "def dir_crawler(path):\n",
        "    if os.path.isfile(path + \"dirs.txt\"):\n",
        "        dirsFile = open(path + \"dirs.txt\", 'r')\n",
        "        for dir in dirsFile:\n",
        "            dir_crawler(path + dir.strip() + \"/\")\n",
        "\n",
        "        dirsFile.close()\n",
        "\n",
        "    if os.path.isfile(path + \"links.txt\"):\n",
        "        linksFile = open(path + \"links.txt\", 'r')\n",
        "        for link in linksFile:\n",
        "            download_url = link.strip()\n",
        "            file_name = urllib.parse.unquote(download_url.rsplit('/', 1)[1])\n",
        "            if os.path.isfile(path+file_name):\n",
        "                print(\"\\\"\"+file_name+\"\\\"\"+\" already downloaded\")\n",
        "                continue\n",
        "            file_downloader(download_url, path, file_name)\n",
        "        linksFile.close()\n",
        "\n",
        "\n",
        "\n",
        "URL = input(\"Enter the URL: \")\n",
        "if URL[-1]!=\"/\":\n",
        "  URL = URL + \"/\"\n",
        "password = input(\"Enter password if any. Else press enter: \")\n",
        "path = input(\"Enter the path to empty directory: \")\n",
        "\n",
        "if path[-1]!='/':\n",
        "  path = path + \"/\"\n",
        "\n",
        "options = webdriver.ChromeOptions()\n",
        "options.add_argument('--headless')\n",
        "options.add_argument('--no-sandbox')\n",
        "options.add_argument('--disable-dev-shm-usage')\n",
        "driver = webdriver.Chrome('chromedriver',options=options)\n",
        "\n",
        "if password:\n",
        "  try:\n",
        "      print(\"\\n\\n::Checking Authentication::\")\n",
        "      driver.get(URL)\n",
        "      WebDriverWait(driver,5).until(EC.alert_is_present(),'Timed out waiting for alerts to appear')\n",
        "      obj = driver.switch_to.alert\n",
        "      obj.send_keys(password)\n",
        "      obj.accept()\n",
        "      print(\"\\n::Authentication Successful::\")\n",
        "  except:\n",
        "      print(\"\\n::Authentication Error::\")\n",
        "\n",
        "\n",
        "print(\"████████████████████████████████████████████\\n\\n\\n::Fetching Links & Creating Folders::\\n\")\n",
        "\n",
        "\n",
        "domain = URL.split(\".dev\")[0] + \".dev\" #get domain name from URL\n",
        "getLinks(URL, path)\n",
        "driver.quit()\n",
        "print(\"\\n::Link Fetch and Folders Creation Complete::\")\n",
        "\n",
        "print(\"████████████████████████████████████████████\\n\\n\\n::Downloading::\\n\")\n",
        "chunkSize = 1024\n",
        "try:\n",
        "    dir_crawler(path)\n",
        "    print(\"\\n\\n::Download Complete::\\n████████████████████████████████████████████\")\n",
        "except:\n",
        "    print(\"\\n\\n::Download Error::\\n████████████████████████████████████████████\")\n"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}