QS-ranking.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "install finished\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "import time\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "\n",
    "print(\"install finished\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n",
      "['Overall Score', 'International Students Ratio', 'International Faculty Ratio', 'Faculty Student Ratio', 'Citations per Faculty', 'Academic Reputation', 'Employer Reputation']\n"
     ]
    }
   ],
   "source": [
    "indicators = [\"Overall Score\", \"International Students Ratio\", \"International Faculty Ratio\", \n",
    "            \"Faculty Student Ratio\", \"Citations per Faculty\", \"Academic Reputation\", \"Employer Reputation\"]\n",
    "print(len(indicators))\n",
    "print(indicators)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_indicator_scores(soup):\n",
    "    items = soup.select(\".td-wrap-in\")\n",
    "    school, scores = [], [[] for i in range(7)]\n",
    "\n",
    "    if(len(items) % 8 != 0):\n",
    "        return [], []\n",
    "\n",
    "    item =[]\n",
    "    for index in range(len(items)):\n",
    "        \n",
    "        if(index % 8 == 0):\n",
    "            item.append(items[index].contents[0].contents[0])\n",
    "        else:\n",
    "            try:\n",
    "                test = float(items[index].contents[0])\n",
    "                item.append(items[index].contents[0])\n",
    "            except:\n",
    "                item.append(\"NULL\")  # NULL score\n",
    "\n",
    "        if(index % 8 == 7):\n",
    "            # append row to list\n",
    "            school.append(item[0])\n",
    "            if(len(item) != 8):\n",
    "                print(item)\n",
    "            for i in range(len(scores)):\n",
    "                scores[i].append(item[i+1])\n",
    "\n",
    "            item =[]\n",
    "            continue\n",
    "    \n",
    "    return school, scores\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_rank(soup):\n",
    "    rows = soup.select(\"._univ-rank.show-this-in-mobile-indi\")\n",
    "    ranks = []\n",
    "    for row in rows:\n",
    "        ranks.append(row.contents[0])\n",
    "    return ranks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "def write_df_csv(rank, school, scores):\n",
    "    df = pd.DataFrame()\n",
    "\n",
    "    if(len(rank) != len(school)):\n",
    "        print(\"length error:\", len(rank), len(school))\n",
    "        return False\n",
    "\n",
    "    df[\"Rank\"] = rank\n",
    "    df[\"University\"] = school\n",
    "    \n",
    "    for index in range(len(indicators)):\n",
    "        df[indicators[index]] = scores[index]\n",
    "        \n",
    "    df.to_csv(\"QS-2022-ranking.csv\", mode='a', header=False, index=False, encoding=\"utf-8\")\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finish csv init\n",
      "current page: 2\n",
      "current page: 2\n",
      "current page: 3\n",
      "current page: 4\n",
      "current page: 5\n",
      "current page: 6\n",
      "current page: 7\n",
      "current page: 8\n",
      "current page: 9\n",
      "current page: 10\n",
      "current page: 11\n",
      "current page: 12\n",
      "current page: 13\n",
      "current page: 14\n",
      "current page: 15\n",
      "current page: 16\n",
      "current page: 17\n",
      "current page: 18\n",
      "current page: 19\n",
      "current page: 20\n",
      "current page: 21\n",
      "current page: 22\n",
      "current page: 23\n",
      "current page: 24\n",
      "current page: 25\n",
      "current page: 26\n",
      "current page: 27\n",
      "current page: 28\n",
      "current page: 29\n",
      "current page: 30\n",
      "current page: 31\n",
      "current page: 32\n",
      "current page: 33\n",
      "current page: 34\n",
      "current page: 35\n",
      "current page: 36\n",
      "current page: 37\n",
      "current page: 38\n",
      "current page: 39\n",
      "current page: 40\n",
      "current page: 41\n",
      "current page: 42\n",
      "current page: 43\n",
      "current page: 44\n",
      "current page: 45\n",
      "current page: 46\n",
      "current page: 47\n",
      "current page: 48\n",
      "current page: 49\n",
      "current page: 50\n",
      "current page: 51\n",
      "current page: 52\n",
      "current page: 53\n",
      "current page: 54\n",
      "current page: 55\n",
      "current page: 56\n",
      "current page: 57\n",
      "current page: 58\n",
      "current page: 59\n",
      "current page: 60\n",
      "current page: 61\n",
      "current page: 62\n",
      "current page: 63\n",
      "current page: 64\n",
      "current page: 65\n",
      "current page: 66\n",
      "current page: 67\n",
      "current page: 68\n",
      "current page: 69\n",
      "current page: 70\n",
      "current page: 71\n",
      "current page: 72\n",
      "current page: 73\n",
      "current page: 74\n",
      "current page: 75\n",
      "current page: 76\n",
      "current page: 77\n",
      "current page: 78\n",
      "current page: 79\n",
      "current page: 80\n",
      "current page: 81\n",
      "current page: 82\n",
      "current page: 83\n",
      "current page: 84\n",
      "current page: 85\n",
      "current page: 86\n",
      "current page: 87\n",
      "current page: 88\n",
      "current page: 89\n",
      "current page: 90\n",
      "current page: 91\n",
      "current page: 92\n",
      "current page: 93\n",
      "current page: 94\n",
      "current page: 95\n",
      "current page: 96\n",
      "current page: 97\n",
      "current page: 98\n",
      "current page: 99\n",
      "current page: 100\n",
      "current page: 101\n",
      "current page: 102\n",
      "current page: 103\n",
      "current page: 104\n",
      "current page: 105\n",
      "current page: 106\n",
      "current page: 107\n",
      "current page: 108\n",
      "current page: 109\n",
      "current page: 110\n",
      "current page: 111\n",
      "current page: 112\n",
      "current page: 113\n",
      "current page: 114\n",
      "current page: 115\n",
      "current page: 116\n",
      "current page: 117\n",
      "current page: 118\n",
      "current page: 119\n",
      "current page: 120\n",
      "current page: 121\n",
      "current page: 122\n",
      "current page: 123\n",
      "current page: 124\n",
      "current page: 125\n",
      "current page: 126\n",
      "current page: 127\n",
      "current page: 128\n",
      "current page: 129\n",
      "current page: 130\n",
      "current page: 131\n",
      "- finish -\n"
     ]
    }
   ],
   "source": [
    "file = open(\"QS-2022-ranking.csv\", 'w')\n",
    "file.write(\"Rank,University,\")\n",
    "for index in range(len(indicators)):\n",
    "    file.write(indicators[index] + \",\")\n",
    "file.write(\"\\n\")\n",
    "file.close()\n",
    "print(\"finish csv init\")\n",
    "\n",
    "option = Options()\n",
    "option.add_argument(\"--disable-notifications\")\n",
    "\n",
    "chrome = webdriver.Chrome('./chromedriver', options=option)\n",
    "url = \"https://www.topuniversities.com/university-rankings/world-university-rankings/2022\"\n",
    "chrome.get(url)\n",
    "time.sleep(2)\n",
    "\n",
    "# click ranking indicator tag\n",
    "chrome.find_element_by_link_text(\"Rankings indicators\", ).send_keys(Keys.ENTER)\n",
    "time.sleep(3)\n",
    "\n",
    "print(\"current page:\", 1)\n",
    "soup = BeautifulSoup(chrome.page_source, 'html.parser')\n",
    "school, scores = get_indicator_scores(soup)\n",
    "ranks = get_rank(soup)\n",
    "result = write_df_csv(ranks, school, scores)\n",
    "if(result == False):\n",
    "    print(\"END with ERROR\")\n",
    "\n",
    "current_page = 2\n",
    "while(True):\n",
    "    print(\"current page:\", current_page)\n",
    "    try:\n",
    "        # click next page\n",
    "        chrome.find_element_by_class_name(\"next\").send_keys(Keys.ENTER)\n",
    "    except:\n",
    "        break\n",
    "    \n",
    "    time.sleep(3)\n",
    "    soup = BeautifulSoup(chrome.page_source, 'html.parser')\n",
    "    \n",
    "    school, scores = get_indicator_scores(soup)\n",
    "    ranks = get_rank(soup)\n",
    "\n",
    "    # write to csv\n",
    "    result = write_df_csv(ranks, school, scores)\n",
    "    if(result == False):\n",
    "        break\n",
    "    current_page += 1\n",
    "\n",
    "chrome.close()\n",
    "print(\"- finish -\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Massachusetts Institute of Technology (MIT) ', 'University of Oxford', 'Stanford University', 'University of Cambridge', 'Harvard University', 'California Institute of Technology (Caltech)', 'Imperial College London', 'ETH Zurich - Swiss Federal Institute of Technology', 'UCL', 'University of Chicago']\n"
     ]
    }
   ],
   "source": [
    "print(school)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
  },
  "kernelspec": {
   "display_name": "Python 3.9.5 64-bit",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": ""
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}