Dataset preparation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "%matplotlib inline\n",
    "%config InlineBackend.figure_formats=['svg']\n",
    "\n",
    "import json\n",
    "import codecs\n",
    "import os\n",
    "import math\n",
    "\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import datetime\n",
    "\n",
    "import pandas as pd\n",
    "import datetime as dt\n",
    "\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.dates as dates\n",
    "import hashlib\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "from collections import Counter\n",
    "import networkx as nx\n",
    "from networkx.algorithms import bipartite \n",
    "\n",
    "from wekeypedia.wikipedia.page import WikipediaPage as Page"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Dataset directory\n",
    "\n",
    "To choose the dataset configure the variable 'dataset_dir'. The directory must contain a file named 'pagenames', which contains one title on each line. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dataset_dir_name = 'listgeometry'\n",
    "print_v = True\n",
    "print_t = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of pages in file: 299\n"
     ]
    }
   ],
   "source": [
    "file_page_names = \"%s/pagenames\" % dataset_dir_name\n",
    "list_of_page_names = [x.strip() for x in codecs.open(file_page_names,\"r\",\"utf-8\").readlines()]\n",
    "\n",
    "if (print_t): print 'Number of pages in file:',len(list_of_page_names) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Basic statistics computation\n",
    "##Data gathering\n",
    "\n",
    "The following functions can be used to download and store the data in files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_path_to_file(name):\n",
    "    hashmd5 = hashlib.md5(name.encode('utf-8')).hexdigest()\n",
    "    path = '/'+hashmd5[0]+'/'+hashmd5[1]+'/'+hashmd5[2]+'/'+hashmd5[3]+'/'\n",
    "    return path,hashmd5\n",
    "\n",
    "def load_pages_data(dataset_dir_name, list_of_page_names):\n",
    "    data_pages_dir_name = '%s/data/pages/' % dataset_dir_name\n",
    "    if not(os.path.exists(data_pages_dir_name)): os.makedirs(data_pages_dir_name)\n",
    "\n",
    "    pages_data = {}\n",
    "    for page_name in list_of_page_names:\n",
    "        path,hashmd5 = get_path_to_file(page_name)\n",
    "        if not(os.path.exists( '%s/%s' % (data_pages_dir_name, path) )):\n",
    "            os.makedirs( '%s/%s' % (data_pages_dir_name, path) )\n",
    "        file_name = '%s/%s/%s.json'%(data_pages_dir_name, path, hashmd5)\n",
    "        if (os.path.exists(file_name)):\n",
    "            with open(file_name) as f:\n",
    "                pages_data[page_name] = json.load(f)\n",
    "        else:\n",
    "            pass\n",
    "    return pages_data\n",
    "\n",
    "\n",
    "def gather_pages_data(dataset_dir_name, list_of_page_names):\n",
    "    data_pages_dir_name = '%s/data/pages/' % dataset_dir_name\n",
    "    if not(os.path.exists(data_pages_dir_name)): os.makedirs(data_pages_dir_name)\n",
    "\n",
    "    pages_data = {}\n",
    "    for page_name in list_of_page_names:\n",
    "        path,hashmd5 = get_path_to_file(page_name)\n",
    "        if not(os.path.exists( '%s/%s' % (data_pages_dir_name, path) )):\n",
    "            os.makedirs( '%s/%s' % (data_pages_dir_name, path) )\n",
    "        file_name = '%s/%s/%s.json'%(data_pages_dir_name, path, hashmd5)\n",
    "        if (os.path.exists(file_name)):\n",
    "            with open(file_name) as f:\n",
    "                pages_data[page_name] = json.load(f)\n",
    "        else:\n",
    "            data = {}\n",
    "            wikipage = Page(title=page_name)\n",
    "            request = wikipage.fetch_info(page_name)['query']['pages']\n",
    "            page_id = list(request)[0]\n",
    "            if page_id!='-1':\n",
    "                try:\n",
    "                    for x in request[page_id]:\n",
    "                        data[x]=request[page_id][x]\n",
    "                    data['revisions']=wikipage.get_revisions_list()\n",
    "                    data['links']= wikipage.get_links()\n",
    "                    data['categories']= wikipage.get_categories()\n",
    "                    pages_data[page_name]=data\n",
    "                    f = open(file_name,'w')\n",
    "                    f.write(json.dumps(data))\n",
    "                    f.close()\n",
    "                except Exception as e:\n",
    "                    print 'Error with page:',page_name\n",
    "                    print e\n",
    "    return(pages_data)\n",
    "\n",
    "## ## ## ## ## ## ## ## ## ##\n",
    "\n",
    "data_pages_dir_name='%s/data/pages/'% (dataset_dir_name)\n",
    "if not(os.path.exists(data_pages_dir_name)): os.mkdir(data_pages_dir_name)\n",
    "\n",
    "pages_data = load_pages_data(list_of_page_names, data_pages_dir_name)\n",
    "if (print_t): print 'Number of pages load:',len(pages_data)\n",
    "\n",
    "if len (list_of_page_names) != len(pages_data.keys()):\n",
    "    if (print_t): print 'Failure to load:'\n",
    "    for fail_page in list_of_page_names:\n",
    "        if fail_page not in pages_data.keys():\n",
    "            if (print_t): print '  >',fail_page\n",
    "    if (print_t): print 'Updating list of page names without them'\n",
    "    list_of_page_names = pages_data.keys()\n",
    "\n",
    "if (print_v): print '> Pages load in: pages_data'\n",
    "    \n",
    "#talk_pages_data=load_pages_data([''.join(['Talk:',x]) for x in list_of_page_names],data_pages_dir_name)  \n",
    "#print 'Number of talk pages load:',len(talk_pages_data)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##Statistics computation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Dataframe of basic stat: df_basic_stats\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Page_id</th>\n",
       "      <th>Length</th>\n",
       "      <th>Namespace</th>\n",
       "      <th>Nb_revisions</th>\n",
       "      <th>Nb_revisions_IP</th>\n",
       "      <th>Nb_revisions_Bot</th>\n",
       "      <th>Nb_revisions_wiki</th>\n",
       "      <th>Nb_editors</th>\n",
       "      <th>Nb_editors_IP</th>\n",
       "      <th>Nb_editors_Bot</th>\n",
       "      <th>Nb_editors_wiki</th>\n",
       "      <th>Links</th>\n",
       "      <th>Date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Digital geometry</th>\n",
       "      <td> 386413</td>\n",
       "      <td>  7211</td>\n",
       "      <td> 0</td>\n",
       "      <td> 116</td>\n",
       "      <td>  51</td>\n",
       "      <td> 11</td>\n",
       "      <td>  54</td>\n",
       "      <td>  63</td>\n",
       "      <td> 16</td>\n",
       "      <td>  7</td>\n",
       "      <td> 40</td>\n",
       "      <td>  46</td>\n",
       "      <td> 1052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Synthetic geometry</th>\n",
       "      <td> 267484</td>\n",
       "      <td> 11870</td>\n",
       "      <td> 0</td>\n",
       "      <td> 129</td>\n",
       "      <td>  14</td>\n",
       "      <td> 11</td>\n",
       "      <td> 104</td>\n",
       "      <td>  60</td>\n",
       "      <td> 11</td>\n",
       "      <td>  5</td>\n",
       "      <td> 44</td>\n",
       "      <td>  98</td>\n",
       "      <td>  910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Triangle inequality</th>\n",
       "      <td>  53941</td>\n",
       "      <td> 25011</td>\n",
       "      <td> 0</td>\n",
       "      <td> 395</td>\n",
       "      <td> 122</td>\n",
       "      <td> 28</td>\n",
       "      <td> 245</td>\n",
       "      <td> 199</td>\n",
       "      <td> 88</td>\n",
       "      <td> 12</td>\n",
       "      <td> 99</td>\n",
       "      <td>  87</td>\n",
       "      <td>  498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Deltahedron</th>\n",
       "      <td> 493995</td>\n",
       "      <td> 13811</td>\n",
       "      <td> 0</td>\n",
       "      <td> 197</td>\n",
       "      <td>  18</td>\n",
       "      <td> 19</td>\n",
       "      <td> 160</td>\n",
       "      <td>  69</td>\n",
       "      <td> 13</td>\n",
       "      <td>  5</td>\n",
       "      <td> 51</td>\n",
       "      <td>  70</td>\n",
       "      <td> 1139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Isoperimetric inequality</th>\n",
       "      <td> 326182</td>\n",
       "      <td> 19249</td>\n",
       "      <td> 0</td>\n",
       "      <td> 176</td>\n",
       "      <td>  28</td>\n",
       "      <td> 27</td>\n",
       "      <td> 121</td>\n",
       "      <td>  94</td>\n",
       "      <td> 21</td>\n",
       "      <td> 12</td>\n",
       "      <td> 61</td>\n",
       "      <td> 105</td>\n",
       "      <td>  982</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          Page_id  Length  Namespace  Nb_revisions  \\\n",
       "Digital geometry           386413    7211          0           116   \n",
       "Synthetic geometry         267484   11870          0           129   \n",
       "Triangle inequality         53941   25011          0           395   \n",
       "Deltahedron                493995   13811          0           197   \n",
       "Isoperimetric inequality   326182   19249          0           176   \n",
       "\n",
       "                          Nb_revisions_IP  Nb_revisions_Bot  \\\n",
       "Digital geometry                       51                11   \n",
       "Synthetic geometry                     14                11   \n",
       "Triangle inequality                   122                28   \n",
       "Deltahedron                            18                19   \n",
       "Isoperimetric inequality               28                27   \n",
       "\n",
       "                          Nb_revisions_wiki  Nb_editors  Nb_editors_IP  \\\n",
       "Digital geometry                         54          63             16   \n",
       "Synthetic geometry                      104          60             11   \n",
       "Triangle inequality                     245         199             88   \n",
       "Deltahedron                             160          69             13   \n",
       "Isoperimetric inequality                121          94             21   \n",
       "\n",
       "                          Nb_editors_Bot  Nb_editors_wiki  Links  Date  \n",
       "Digital geometry                       7               40     46  1052  \n",
       "Synthetic geometry                     5               44     98   910  \n",
       "Triangle inequality                   12               99     87   498  \n",
       "Deltahedron                            5               51     70  1139  \n",
       "Isoperimetric inequality              12               61    105   982  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def stat_computation(pages_data):\n",
    "    df = pd.DataFrame(index=pages_data.keys())\n",
    "    \n",
    "    #pageid\n",
    "    data={k:v['pageid'] for k,v in pages_data.items()}\n",
    "    for k in pages_data.keys(): df.ix[k,'Page_id'] = data[k]\n",
    "    #length\n",
    "    data={k:v['length'] for k,v in pages_data.items()}\n",
    "    for k in pages_data.keys(): df.ix[k,'Length'] = data[k]\n",
    "    #namespace\n",
    "    data={k:v['ns'] for k,v in pages_data.items()}\n",
    "    for k in pages_data.keys(): df.ix[k,'Namespace'] = data[k]\n",
    "    #nombre de revisions\n",
    "    data={k:len(v['revisions']) for k,v in pages_data.items()}\n",
    "    for k in pages_data.keys(): df.ix[k,'Nb_revisions'] = data[k]\n",
    "    #nombre de revisions by IP\n",
    "    data={k:len([x for x in v['revisions'] if ('userid' in x and x['userid']==0)])\n",
    "            for k,v in pages_data.items()}\n",
    "    for k in pages_data.keys(): df.ix[k,'Nb_revisions_IP'] = data[k]\n",
    "    #nombre de revisions by Bot\n",
    "    data={k:len([x for x in v['revisions'] if ('user' in x and 'bot' in x['user'].lower())])\n",
    "            for k,v in pages_data.items()}\n",
    "    for k in pages_data.keys(): df.ix[k,'Nb_revisions_Bot'] = data[k]\n",
    "    #nombre de revisions by Alive Registered Members\n",
    "    data={k:len([x for x in v['revisions'] if ('user' in x and x['userid']!=0 and 'bot' not in x['user'].lower())])\n",
    "            for k,v in pages_data.items()}\n",
    "    for k in pages_data.keys(): df.ix[k,'Nb_revisions_wiki'] = data[k]\n",
    "    #nombre de contributeurs\n",
    "    data={k:len(set([x['user'] for x in v['revisions'] if 'user' in x]))\n",
    "            for k,v in pages_data.items()}\n",
    "    for k in pages_data.keys(): df.ix[k,'Nb_editors'] = data[k]\n",
    "    #nombre de contributeurs IP\n",
    "    data={k:len(set([x['user'] for x in v['revisions'] if 'userid' in x and x['userid']==0]))\n",
    "            for k,v in pages_data.items()}\n",
    "    for k in pages_data.keys(): df.ix[k,'Nb_editors_IP'] = data[k]\n",
    "    #nombre de contributeurs Bot\n",
    "    data={k:len(set([x['user'] for x in v['revisions'] if 'user' in x and 'bot' in x['user']]))\n",
    "            for k,v in pages_data.items()}\n",
    "    for k in pages_data.keys(): df.ix[k,'Nb_editors_Bot'] = data[k]\n",
    "    #nombre de contributeurs by Alive Registered Members\n",
    "    data={k:len(set([x['user'] for x in v['revisions'] if 'user' in x and x['userid']!=0 and 'bot' not in x['user']]))\n",
    "            for k,v in pages_data.items()}\n",
    "    for k in pages_data.keys(): df.ix[k,'Nb_editors_wiki'] = data[k]\n",
    "    #nombre de revisions\n",
    "    data={k:len(v['links']) for k,v in pages_data.items()}\n",
    "    for k in pages_data.keys(): df.ix[k,'Links'] = data[k]\n",
    "    #date of the first contibutions (in number of days after the start of the wikipedia project)\n",
    "    def numberOfDaysAfter(date):\n",
    "        return( (datetime.datetime.strptime(date,\"%Y-%m-%dT%H:%M:%SZ\")-datetime.datetime.strptime(\"2001-01-15T00:00:00Z\",\"%Y-%m-%dT%H:%M:%SZ\")).days)\n",
    "    data={k:min(map(numberOfDaysAfter,map(lambda x: x['timestamp'],v['revisions'])))  for k,v in pages_data.items()}\n",
    "    for k in pages_data.keys(): df.ix[k,'Date'] = data[k]\n",
    "\n",
    "    return(df)\n",
    "\n",
    "## ## ## ## ## ## ## ## ## ##\n",
    "\n",
    "stats_dir_name = \"%s/stats/\" % dataset_dir_name\n",
    "if not(os.path.exists(stats_dir_name)): os.mkdir(stats_dir_name)\n",
    "\n",
    "basic_stats_file_name = '%s/basic_stats.csv' % stats_dir_name\n",
    "\n",
    "df_basic_stats = pd.DataFrame()\n",
    "\n",
    "if ( os.path.exists(basic_stats_file_name) ):\n",
    "    df_basic_stats = df_basic_stats.from_csv(basic_stats_file_name, encoding=\"utf-8\")\n",
    "else:\n",
    "    df_basic_stats = stat_computation(pages_data)\n",
    "    df_basic_stats.to_csv(basic_stats_file_name, encoding=\"utf-8\")\n",
    "\n",
    "if (print_v): print '> Dataframe of basic stat: df_basic_stats'\n",
    "df_basic_stats.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Statistique on content of the last revision\n",
    "\n",
    "## Gathering last revision content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Text of pages store in: pages_last_revision_text\n"
     ]
    }
   ],
   "source": [
    "def load_pages_last_revision_text(list_of_page_names, text_dir_name):\n",
    "    last_revision_text = {}\n",
    "    \n",
    "    for page_name in list_of_page_names:\n",
    "        file_name = '%s/%s.html' % (text_dir_name,page_name)\n",
    "        if (os.path.exists(file_name)):\n",
    "            last_revision_text[page_name] = ''.join(codecs.open(file_name,\"r\", \"utf-8-sig\").readlines())\n",
    "        else:\n",
    "            data = {}\n",
    "            wikipage = Page(title=page_name)\n",
    "            last_revision_text[page_name] = wikipage.get_current()\n",
    "            f = open(file_name,'w')\n",
    "            f.write(last_revision_text[page_name].encode('utf-8'))\n",
    "            f.close()\n",
    "                \n",
    "    return(last_revision_text)\n",
    "\n",
    "## ## ## ## ## ## ## ## ## ##\n",
    "\n",
    "text_dir_name = '%s/data/text/' % dataset_dir_name\n",
    "if not(os.path.exists(text_dir_name)): os.mkdir(text_dir_name)\n",
    "\n",
    "pages_last_revision_text = load_pages_last_revision_text(list_of_page_names, text_dir_name)\n",
    "\n",
    "if (print_t): print 'Text of the last revision of ', len(pages_data),'pages load.'\n",
    "if (print_v): print '> Text of pages store in: pages_last_revision_text'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Computation of statistics on content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Dataframe of word basic stat: df_word_stats\n"
     ]
    }
   ],
   "source": [
    "def basic_word_analysis(pages_text_last_revision):\n",
    "    df = pd.DataFrame(index=pages_text_last_revision.keys())\n",
    "\n",
    "    for page_name in pages_text_last_revision:\n",
    "        text = BeautifulSoup( pages_text_last_revision[page_name] ).text\n",
    "        words = text.split(\" \")\n",
    "        df.ix[page_name,'Nb_words'] = len(words)\n",
    "        df.ix[page_name,'Average_word_length'] = sum([len(x) for x in words])/float(len(words))   \n",
    "\n",
    "    return df\n",
    "\n",
    "## ## ## ## ## ## ## ## ## ##\n",
    "\n",
    "words_stats_file_name = '%s/words_stats.csv' % stats_dir_name\n",
    "\n",
    "df_word_stats = pd.DataFrame()\n",
    "\n",
    "if os.path.exists(words_stats_file_name):\n",
    "    df_word_stats = df_word_stats.from_csv(words_stats_file_name, encoding=\"utf-8\")\n",
    "else:\n",
    "    df_word_stats = basic_word_analysis(pages_last_revision_text)\n",
    "    df_word_stats.to_csv(words_stats_file_name, encoding=\"utf-8\")\n",
    "\n",
    "df_word_stats.head(5)\n",
    "\n",
    "if (print_v): print '> Dataframe of word basic stat: df_word_stats'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Pageviews time series\n",
    "\n",
    "##Gather pages views"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def gather_pages_views(list_of_page_names, pages_views_dir_name):\n",
    "    error = []\n",
    "    for page_name in list_of_page_names:\n",
    "        path,hashmd5 = get_path_to_file(page_name)\n",
    "        file_name = '%s/%s/%s.json' % (pages_views_dir_name,path, hashmd5)\n",
    "        if not (os.path.exists(file_name)):            \n",
    "            try:\n",
    "                wikipage = Page(title=page_name)\n",
    "                page_views_ts = {k:v for x in wikipage.get_pageviews() for (k,v) in x.items()}\n",
    "                f = open(filename,'w')\n",
    "                f.write(json.dumps(page_views_ts))\n",
    "                f.close()\n",
    "            except:\n",
    "                error.append(page_name)\n",
    "    return(error)\n",
    "\n",
    "## ## ## ## ## ## ## ## ## ##\n",
    "\n",
    "pages_views_dir_name = '%s/data/pagesviews/' % dataset_dir_name\n",
    "if not(os.path.exists(pages_views_dir_name)): os.mkdir(pages_views_dir_name)\n",
    "\n",
    "error = gather_pages_views(list_of_page_names, pages_views_dir_name)\n",
    "while len(error)>0:\n",
    "    error = gather_pages_views(error, pages_views_dir_name)\n",
    "\n",
    "#print '!'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##Compute pages views time series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Dic of dataframes of daily pageviews time series: pages_views_daily_ts\n",
      "> Dic of dataframes of weekly pageviews time series: pages_views_weekly_ts\n",
      "> Dic of dataframes of monthly pageviews time series: pages_views_monthly_ts\n"
     ]
    }
   ],
   "source": [
    "def get_pages_views_time_series(list_of_page_names, pages_views_dir_name, time_series_dir_name):\n",
    "    pages_views_daily_ts={}\n",
    "    pages_views_weekly_ts={}       \n",
    "    pages_views_monthly_ts={}\n",
    "    for page_name in list_of_page_names:\n",
    "        path,hashmd5 = get_path_to_file(page_name)\n",
    "        file_name = '%s/%s/%s.json' % (pages_views_dir_name,path, hashmd5)\n",
    "        if not(os.path.exists( '%s/%s' % (time_series_dir_name,path) )):\n",
    "            os.makedirs( '%s/%s' % (time_series_dir_name,path) )\n",
    "        file_name_daily = '%s/%s/%s.pageviews.daily.csv' % (time_series_dir_name,path,page_name)\n",
    "        file_name_weekly = '%s/%s/%s.pageviews.weekly.csv' % (time_series_dir_name,path,page_name)\n",
    "        file_name_monthly = '%s/%s/%s.pageviews.monthly.csv' % (time_series_dir_name,path,page_name)\n",
    "        if os.path.exists(file_name):\n",
    "            if (os.path.exists(file_name_daily)):\n",
    "                pages_views_daily_ts[page_name] = pd.DataFrame().from_csv(file_name_daily)\n",
    "                pages_views_weekly_ts[page_name] = pd.DataFrame().from_csv(file_name_weekly)\n",
    "                pages_views_monthly_ts[page_name] = pd.DataFrame().from_csv(file_name_monthly)\n",
    "            else:\n",
    "                data=json.load(open(file_name)).items()\n",
    "                index = []\n",
    "                series = []\n",
    "                for k,v in data:\n",
    "                    try:\n",
    "                        index.append(pd.to_datetime(k, format=\"%Y-%m-%d\"))\n",
    "                        series.append(v)\n",
    "                    except ValueError:\n",
    "                        continue\n",
    "                df = pd.DataFrame(series,index=index,columns=['page_views'])\n",
    "                pages_views_daily_ts[page_name]=df\n",
    "                pages_views_daily_ts[page_name].to_csv(file_name_daily,encoding=\"utf-8\")\n",
    "                pages_views_weekly_ts[page_name] = df.resample('W-MON', how='sum')\n",
    "                pages_views_weekly_ts[page_name].to_csv(file_name_weekly,encoding=\"utf-8\")\n",
    "                pages_views_monthly_ts[page_name] = df.resample('M', how='sum')\n",
    "                pages_views_monthly_ts[page_name].to_csv(file_name_monthly,encoding=\"utf-8\")\n",
    "    return(pages_views_daily_ts,pages_views_weekly_ts,pages_views_monthly_ts)\n",
    "\n",
    "## ## ## ## ## ## ## ## ## ##\n",
    "    \n",
    "time_series_dir_name = '%s/stats/time series/' % dataset_dir_name\n",
    "if not(os.path.exists(time_series_dir_name)): os.mkdir(time_series_dir_name)\n",
    "    \n",
    "pages_views_daily_ts,pages_views_weekly_ts,pages_views_monthly_ts = get_pages_views_time_series(list_of_page_names, pages_views_dir_name, time_series_dir_name)\n",
    "\n",
    "if (print_t): print 'Number of daily pageviews time series:',len(pages_views_daily_ts)\n",
    "if (print_v): print '> Dic of dataframes of daily pageviews time series: pages_views_daily_ts'\n",
    "\n",
    "if (print_t): print 'Number of weekly pageviews time series:',len(pages_views_weekly_ts)\n",
    "if (print_v): print '> Dic of dataframes of weekly pageviews time series: pages_views_weekly_ts'\n",
    "\n",
    "if (print_t): print 'Number of monthly pageviews time series:',len(pages_views_monthly_ts)\n",
    "if (print_v): print '> Dic of dataframes of monthly pageviews time series: pages_views_monthly_ts'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Revisions time series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Dic of dataframes of daily revisions time series: revisions_daily_ts\n",
      "> Dic of dataframes of weekly revisions time series: revisions_weekly_ts\n",
      "> Dic of dataframes of monthly revisions time series: revisions_monthly_ts\n",
      "> Dic of dataframes of daily IP revisions time series: revisions_ip_daily_ts\n",
      "> Dic of dataframes of weekly IP revisions time series: revisions_ip_weekly_ts\n",
      "> Dic of dataframes of monthly IP revisions time series: revisions_ip_monthly_ts\n",
      "> Dic of dataframes of daily bot revisions time series: revisions_bot_daily_ts\n",
      "> Dic of dataframes of weekly bot revisions time series: revisions_bot_weekly_ts\n",
      "> Dic of dataframes of monthly bot revisions time series: revisions_bot_monthly_ts\n",
      "> Dic of dataframes of daily members revisions time series: revisions_members_daily_ts\n",
      "> Dic of dataframes of weekly members revisions time series: revisions_members_weekly_ts\n",
      "> Dic of dataframes of monthly members revisions time series: revisions_members_monthly_ts\n"
     ]
    }
   ],
   "source": [
    "def get_pages_revisions_time_series_gen(pages_revisions,suffixe,time_series_dir_name):\n",
    "    revisions_daily_ts={}\n",
    "    revisions_weekly_ts={}       \n",
    "    revisions_monthly_ts={}\n",
    "    for page_name in pages_revisions.keys():\n",
    "        path,hashmd5 = get_path_to_file(page_name)\n",
    "        file_name = '%s/%s/%s.json' % (pages_views_dir_name,path, hashmd5)\n",
    "        if not(os.path.exists( '%s/%s' % (time_series_dir_name,path) )):\n",
    "            os.makedirs( '%s/%s' % (time_series_dir_name,path) )\n",
    "        file_name_daily = '%s/%s/%s.%s.daily.csv'%(time_series_dir_name,path,suffixe,page_name)\n",
    "        file_name_weekly = '%s/%s/%s.%s.weekly.csv'%(time_series_dir_name,path,suffixe,page_name)\n",
    "        file_name_monthly = '%s/%s/%s.%s.monthly.csv'%(time_series_dir_name,path,suffixe,page_name)\n",
    "        if (os.path.exists(file_name_daily)):\n",
    "            revisions_daily_ts[page_name] = pd.DataFrame().from_csv(file_name_daily)\n",
    "            revisions_weekly_ts[page_name] = pd.DataFrame().from_csv(file_name_weekly)\n",
    "            revisions_monthly_ts[page_name] = pd.DataFrame().from_csv(file_name_monthly)\n",
    "        else:\n",
    "            revisions=pages_revisions[page_name]\n",
    "            \n",
    "            if len(revisions)==0:\n",
    "                #print 'No revisions for %s with suffixe: %s' % (page_name,suffixe)\n",
    "                continue\n",
    "            \n",
    "            df = pd.DataFrame(revisions)\n",
    "            df[\"datetime\"] = df.timestamp.apply(pd.to_datetime)\n",
    "            df[\"day\"] = df.datetime.apply(dt.date.strftime, args=('%Y-%m-%d',))\n",
    "            counts = df.groupby(df[\"day\"])\n",
    "            counts = counts.aggregate(len)\n",
    "            series = counts[\"size\"].tolist()\n",
    "            index = counts.index.map(lambda x: pd.to_datetime(x, format=\"%Y-%m-%d\"))\n",
    "            df = pd.DataFrame(series,columns=['revisions'],index=index)\n",
    "            \n",
    "            revisions_daily_ts[page_name]=df\n",
    "            revisions_daily_ts[page_name]=revisions_daily_ts[page_name].fillna(0)\n",
    "            revisions_daily_ts[page_name].to_csv(file_name_daily,encoding=\"utf-8\")\n",
    "            \n",
    "            revisions_weekly_ts[page_name]=df.resample('W-MON', how='sum')\n",
    "            revisions_weekly_ts[page_name]=revisions_weekly_ts[page_name].fillna(0)           \n",
    "            revisions_weekly_ts[page_name].to_csv(file_name_weekly,encoding=\"utf-8\")\n",
    "\n",
    "            revisions_monthly_ts[page_name]=df.resample('M', how='sum')\n",
    "            revisions_monthly_ts[page_name]=revisions_monthly_ts[page_name].fillna(0)\n",
    "            revisions_monthly_ts[page_name].to_csv(file_name_monthly,encoding=\"utf-8\")\n",
    "    \n",
    "    return(revisions_daily_ts,revisions_weekly_ts,revisions_monthly_ts)\n",
    "\n",
    "\n",
    "## ## ## ## ## ## ## ## ## ##\n",
    "\n",
    "\n",
    "def get_pages_revisions_time_series(pages_data,time_series_dir_name):\n",
    "    data = {}\n",
    "    for k,v in pages_data.iteritems():\n",
    "        data[k]=v['revisions']\n",
    "    return get_pages_revisions_time_series_gen(data,'revisions',time_series_dir_name)\n",
    "\n",
    "def get_pages_ip_revisions_time_series(pages_data,time_series_dir_name):\n",
    "    data = {}\n",
    "    for k,v in pages_data.iteritems():\n",
    "        data[k]=[x for x in v['revisions'] if ('userid' in x and x['userid']==0)]\n",
    "    return get_pages_revisions_time_series_gen(data,'revisions.ip',time_series_dir_name)\n",
    "\n",
    "def get_pages_bot_revisions_time_series(pages_data,time_series_dir_name):\n",
    "    data = {}\n",
    "    for k,v in pages_data.iteritems():\n",
    "        data[k]=[x for x in v['revisions'] if ('user' in x and 'bot' in x['user'].lower())]\n",
    "    return get_pages_revisions_time_series_gen(data,'revisions.bot',time_series_dir_name)\n",
    "\n",
    "def get_pages_members_revisions_time_series(pages_data,time_series_dir_name):\n",
    "    data = {}\n",
    "    for k,v in pages_data.iteritems():\n",
    "        data[k]=[x for x in v['revisions'] if ('user' in x and x['userid']!=0 and 'bot' not in x['user'].lower())]\n",
    "    return get_pages_revisions_time_series_gen(data,'revisions.members',time_series_dir_name)\n",
    "    \n",
    "    \n",
    "## ## ## ## ## ## ## ## ## ##\n",
    "\n",
    "\n",
    "revisions_daily_ts,revisions_weekly_ts,revisions_monthly_ts = get_pages_revisions_time_series(pages_data,time_series_dir_name)\n",
    "revisions_ip_daily_ts,revisions_ip_weekly_ts,revisions_ip_monthly_ts = get_pages_ip_revisions_time_series(pages_data,time_series_dir_name)\n",
    "revisions_bot_daily_ts,revisions_bot_weekly_ts,revisions_bot_monthly_ts = get_pages_bot_revisions_time_series(pages_data,time_series_dir_name)\n",
    "revisions_members_daily_ts,revisions_members_weekly_ts,revisions_members_monthly_ts = get_pages_members_revisions_time_series(pages_data,time_series_dir_name)\n",
    "\n",
    "#revisions_daily_ts,revisions_weekly_ts,revisions_monthly_ts = get_pages_revisions_time_series(pages_data,'revisions',time_series_dir_name)\n",
    "#print pages_data[pages_data.keys()[0]]['revisions']\n",
    "## store the content of the last revisions also in pages_data\n",
    "#for page_name in pages_data:\n",
    "#    pages_data[page_name]['revisions_daily_ts']=revisions_daily_ts[page_name]\n",
    "#    pages_data[page_name]['revisions_weekly_ts']=revisions_weekly_ts[page_name]\n",
    "#    pages_data[page_name]['revisions_monthly_ts']=revisions_monthly_ts[page_name]\n",
    "\n",
    "\n",
    "if (print_t): print 'Number of revisions time series of page load:',len(revisions_daily_ts)\n",
    "if (print_v): print '> Dic of dataframes of daily revisions time series: revisions_daily_ts'\n",
    "if (print_t): print 'Number of weekly pageviews time series:',len(revisions_weekly_ts)\n",
    "if (print_v): print '> Dic of dataframes of weekly revisions time series: revisions_weekly_ts'\n",
    "if (print_t): print 'Number of monthly pageviews time series:',len(revisions_monthly_ts)\n",
    "if (print_v): print '> Dic of dataframes of monthly revisions time series: revisions_monthly_ts'\n",
    "\n",
    "if (print_t): print 'Number of IP revisions time series of page load:',len(revisions_ip_daily_ts)\n",
    "if (print_v): print '> Dic of dataframes of daily IP revisions time series: revisions_ip_daily_ts'\n",
    "if (print_t): print 'Number of IP weekly pageviews time series:',len(revisions_ip_weekly_ts)\n",
    "if (print_v): print '> Dic of dataframes of weekly IP revisions time series: revisions_ip_weekly_ts'\n",
    "if (print_t): print 'Number of IP monthly pageviews time series:',len(revisions_ip_monthly_ts)\n",
    "if (print_v): print '> Dic of dataframes of monthly IP revisions time series: revisions_ip_monthly_ts'\n",
    "\n",
    "if (print_t): print 'Number of bot revisions time series of page load:',len(revisions_bot_daily_ts)\n",
    "if (print_v): print '> Dic of dataframes of daily bot revisions time series: revisions_bot_daily_ts'\n",
    "if (print_t): print 'Number of bot weekly pageviews time series:',len(revisions_bot_weekly_ts)\n",
    "if (print_v): print '> Dic of dataframes of weekly bot revisions time series: revisions_bot_weekly_ts'\n",
    "if (print_t): print 'Number of bot monthly pageviews time series:',len(revisions_bot_monthly_ts)\n",
    "if (print_v): print '> Dic of dataframes of monthly bot revisions time series: revisions_bot_monthly_ts'\n",
    "\n",
    "if (print_t): print 'Number of members revisions time series of page load:',len(revisions_members_daily_ts)\n",
    "if (print_v): print '> Dic of dataframes of daily members revisions time series: revisions_members_daily_ts'\n",
    "if (print_t): print 'Number of members weekly pageviews time series:',len(revisions_members_weekly_ts)\n",
    "if (print_v): print '> Dic of dataframes of weekly members revisions time series: revisions_members_weekly_ts'\n",
    "if (print_t): print 'Number of members monthly pageviews time series:',len(revisions_members_monthly_ts)\n",
    "if (print_v): print '> Dic of dataframes of monthly members revisions time series: revisions_members_monthly_ts'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Graph Construction\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Graph of occurences computation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> graph of occurences store in: occurences_graph\n"
     ]
    }
   ],
   "source": [
    "# Add an occurence of the title in the content of the page\n",
    "def compute_occurences_graph(pages_data, pages_last_revision_text):\n",
    "    occurences_graph = nx.Graph()\n",
    "    \n",
    "    for page_name in pages_data:\n",
    "        occurences_graph.add_node(page_name)\n",
    "    \n",
    "    for page_name in pages_data.keys():\n",
    "        intradomain =  sorted( pages_data.keys(), key=lambda k: -len(k) )\n",
    "        intradomain.remove(page_name)\n",
    "        gruyere = BeautifulSoup(pages_last_revision_text[page_name]).text.lower()\n",
    "        occurences_name = {}\n",
    "        for occu in intradomain:\n",
    "            occurences_name[occu] = unicode(gruyere).count(unicode(occu.lower()))\n",
    "            gruyere = gruyere.replace(occu, \"\")\n",
    "        \n",
    "        #print occurences_name\n",
    "        ave_occurences_name = sum(occurences_name.values())/float(len(occurences_name))\n",
    "\n",
    "        occurences_name = {k:v for k,v in occurences_name.items() if v>ave_occurences_name}\n",
    "     \n",
    "        for occu in occurences_name:\n",
    "            occurences_graph.add_edge(page_name,occu,attr_dict={\n",
    "                    'distance':1/float(1+occurences_name[occu]),\n",
    "                    'weight': occurences_name[occu]})\n",
    "    \n",
    "    return(occurences_graph)\n",
    "\n",
    "## ## ## ## ## ## ## ## ## ##\n",
    "\n",
    "graph_dir_name = '%s/graph/' % (dataset_dir_name)\n",
    "if not(os.path.exists(graph_dir_name)): os.mkdir(graph_dir_name)\n",
    "\n",
    "file_occurences_graph_name = '%s/occurences_graph.gexf' % (graph_dir_name)\n",
    "occurences_graph = nx.Graph()\n",
    "\n",
    "if (os.path.exists(file_occurences_graph_name)):\n",
    "    occurences_graph = nx.read_gexf(file_occurences_graph_name)\n",
    "else:\n",
    "    occurences_graph = compute_occurences_graph(pages_data,pages_last_revision_text)\n",
    "    nx.write_gexf(occurences_graph, file_occurences_graph_name)\n",
    "\n",
    "if (print_t): print 'Graph of occurences:'\n",
    "if (print_t): print '  number of nodes:',len(occurences_graph.nodes())\n",
    "if (print_t): print '  number of edges:',len(occurences_graph.edges())\n",
    "\n",
    "if (print_v): print '> graph of occurences store in: occurences_graph'\n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Graph of links computation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> graph of links store in: links_graph\n"
     ]
    }
   ],
   "source": [
    "# Add an occurence of the title in the content of the page\n",
    "def compute_links_graph(pages_data, pages_last_revision_text):\n",
    "    links_graph = nx.Graph()\n",
    "    \n",
    "    for page_name in pages_data:\n",
    "        links_graph.add_node(page_name)\n",
    "    \n",
    "    for page_name in pages_data.keys():\n",
    "        links = Counter( [x for x in pages_data[page_name]['links'] if x in pages_data.keys()] )                 \n",
    "     \n",
    "        for l in links:\n",
    "            links_graph.add_edge(page_name,l,attr_dict={\n",
    "                    'distance':1/float(1+links[l]),\n",
    "                    'weight': links[l]})\n",
    "    \n",
    "    return(links_graph)\n",
    "\n",
    "## ## ## ## ## ## ## ## ## ##\n",
    "\n",
    "graph_dir_name = '%s/graph/' % (dataset_dir_name)\n",
    "if not(os.path.exists(graph_dir_name)): os.mkdir(graph_dir_name)\n",
    "\n",
    "file_links_graph_name = '%s/links_graph.gexf' % (graph_dir_name)\n",
    "links_graph = nx.Graph()\n",
    "\n",
    "if (os.path.exists(file_links_graph_name)):\n",
    "    links_graph = nx.read_gexf(file_links_graph_name)\n",
    "else:\n",
    "    links_graph = compute_links_graph(pages_data,pages_last_revision_text)\n",
    "    nx.write_gexf(links_graph, file_links_graph_name)\n",
    "\n",
    "if (print_t): print 'Graph of links:'\n",
    "if (print_t): print '  number of nodes:',len(links_graph.nodes())\n",
    "if (print_t): print '  number of edges:',len(links_graph.edges())\n",
    "\n",
    "if (print_v): print '> graph of links store in: links_graph'\n",
    "\n",
    "#components = nx.connected_components(links_graph)\n",
    "#for con in components:\n",
    "#    print 'Size:',len(con)\n",
    "#    print con"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Page/editors bipartite graph computation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> bipartite graph page/editors store in: pages_editors_bipartite_graph\n"
     ]
    }
   ],
   "source": [
    "def compute_pages_editors_bipartite_graph(pages_data):\n",
    "    pages_editors_bipartite_graph = nx.Graph()\n",
    "    editors_all = {}\n",
    "    for page_name in pages_data:\n",
    "        editors = Counter([x['user'] for x in pages_data[page_name]['revisions'] \n",
    "                           if ('user' in x) and (x['userid']!=0) and ('bot' not in x['user'].lower())])\n",
    "        pages_editors_bipartite_graph.add_node(page_name, type='page')\n",
    "        pages_editors_bipartite_graph.node[page_name][\"revisions\"]=len(pages_data[page_name]['revisions'])\n",
    "        pages_editors_bipartite_graph.node[page_name][\"editors\"]=len(editors)\n",
    "        \n",
    "        for e in editors:\n",
    "            if e not in editors_all: \n",
    "                pages_editors_bipartite_graph.add_node(''.join(['editor:',e]), type=\"editor\")\n",
    "                editors_all[e]=editors[e]\n",
    "            else:\n",
    "                editors_all[e]+=editors[e]\n",
    "            pages_editors_bipartite_graph.add_edge(''.join(['editor:',e]), page_name, \n",
    "                                         attr_dict={'revisions':editors[e]})\n",
    "    #add number of revision on editor node\n",
    "    for e in editors_all:\n",
    "         pages_editors_bipartite_graph.node[''.join(['editor:',e])][\"revisions\"]=editors_all[e]\n",
    "    return(pages_editors_bipartite_graph)\n",
    "\n",
    "## ## ## ## ## ## ## ## ## ##\n",
    "\n",
    "file_pages_editors_bipartite_graph_name = '%s/pages_editors_bipartite_graph.gexf' % (graph_dir_name)\n",
    "pages_editors_bipartite_graph = nx.Graph()\n",
    "\n",
    "if (os.path.exists(file_pages_editors_bipartite_graph_name)):\n",
    "    pages_editors_bipartite_graph = nx.read_gexf(file_pages_editors_bipartite_graph_name)\n",
    "else:\n",
    "    pages_editors_bipartite_graph =compute_pages_editors_bipartite_graph(pages_data)\n",
    "    nx.write_gexf(pages_editors_bipartite_graph, file_pages_editors_bipartite_graph_name) \n",
    "\n",
    "if (print_t): print 'Bipartite graph page/editors'\n",
    "if (print_t): print '  number of nodes:',len(pages_editors_bipartite_graph.nodes())\n",
    "if (print_t): print '  number of edges:',len(pages_editors_bipartite_graph.edges())\n",
    "\n",
    "if (print_v): print '> bipartite graph page/editors store in: pages_editors_bipartite_graph'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Projection graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> projected graph page store in: projected_graph_page\n"
     ]
    }
   ],
   "source": [
    "def compute_projected_graph_page(pages_editors_bipartite_graph):\n",
    "    selected = [x[0] for x in pages_editors_bipartite_graph.nodes(data=True) if x[1]['type']=='page']\n",
    "    res=bipartite.projected_graph(pages_editors_bipartite_graph, selected)\n",
    "    for p1 in res.nodes():    \n",
    "        for p2 in res[p1]:\n",
    "            coeditors = set(pages_editors_bipartite_graph[p1]) & set(pages_editors_bipartite_graph[p2])             \n",
    "            editors = pages_editors_bipartite_graph.node[p1]['editors']+pages_editors_bipartite_graph.node[p2]['editors']\n",
    "            res[p1][p2]['weight'] = len(coeditors) \n",
    "            res[p1][p2]['distance'] = 1/float(1+len(coeditors))\n",
    "            \n",
    "    return(res)\n",
    "\n",
    "## ## ## ## ## ## ## ## ## ##\n",
    "\n",
    "file_projected_graph_page_name = '%s/projected_graph_page.gexf' % (graph_dir_name)\n",
    "projected_graph_page = nx.Graph()\n",
    "\n",
    "if (os.path.exists(file_projected_graph_page_name)):\n",
    "    projected_graph_page = nx.read_gexf(file_projected_graph_page_name)\n",
    "else:\n",
    "    projected_graph_page = compute_projected_graph_page(pages_editors_bipartite_graph)\n",
    "    nx.write_gexf(projected_graph_page, file_projected_graph_page_name)   \n",
    "\n",
    "if (print_t): print 'Projection graph on pages'\n",
    "if (print_t): print '  number of nodes:',len(projected_graph_page.nodes())\n",
    "if (print_t): print '  number of edges:',len(projected_graph_page.edges())\n",
    "    \n",
    "if (print_v): print '> projected graph page store in: projected_graph_page'\n",
    "\n",
    "#components = nx.connected_components(projected_graph_page)\n",
    "#for con in components:\n",
    "#    print 'Size:',len(con)\n",
    "#    print con"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Statistics on graph    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> dataframe of statistics on projected graph page node: df_pro_graph_page_node\n",
      "> dataframe of statistics on projected graph page pair of nodes: df_pro_graph_page_pair\n",
      "> dataframe of statistics on links graph node: df_links_graph_node\n",
      "> dataframe of statistics on links graph pair of nodes: df_links_graph_pair\n",
      "done computation\n",
      "> dataframe of statistics on occurences graph node: df_occurences_graph_node\n",
      "> dataframe of statistics on occurences graph pair of nodes: df_occurences_graph_pair\n"
     ]
    }
   ],
   "source": [
    "def compute_graph_statistics_on_nodes(graph,weight=None):\n",
    "    df = pd.DataFrame(index=graph.nodes())\n",
    "    #Degree centrality\n",
    "    try:\n",
    "        data=nx.degree_centrality(graph)\n",
    "        for k in graph.nodes(): df.ix[k,'Degree Cen'] = data[k]\n",
    "    except Exception as e:\n",
    "        print 'Unable to compute degree centrality.'\n",
    "        print 'Error:',e\n",
    "    #Closeness centrality\n",
    "    try:\n",
    "        data=nx.closeness_centrality(graph)\n",
    "        for k in graph.nodes(): df.ix[k,'Closeness Cen'] = data[k]\n",
    "    except Exception as e:\n",
    "        print 'Unable to compute closeness centrality.'\n",
    "        print 'Error:',e\n",
    "    #Betweenness centrality\n",
    "    try:\n",
    "        data=nx.betweenness_centrality(graph, weight=weight)\n",
    "        for k in graph.nodes(): df.ix[k,'Betweenness Cen'] = data[k]\n",
    "    except Exception as e:\n",
    "        print 'Unable to compute betweenness centrality.'\n",
    "        print 'Error:',e\n",
    "    #Current flow closeness centrality\n",
    "    try:\n",
    "        data=nx.current_flow_closeness_centrality(graph, weight=weight)\n",
    "        for k in graph.nodes(): df.ix[k,'Flow Clos Cen'] = data[k]\n",
    "    except Exception as e:\n",
    "        print 'Unable to compute flow closeness centrality.'\n",
    "        print 'Error:',e\n",
    "    #Current flow betweenness centrality\n",
    "    try:\n",
    "        data=nx.current_flow_betweenness_centrality(graph, weight=weight)\n",
    "        for k in graph.nodes(): df.ix[k,'Cur Flow Clos Cen'] = data[k]\n",
    "    except Exception as e:\n",
    "        print 'Unable to compute current flow closeness centrality.'\n",
    "        print 'Error:',e\n",
    "    #Pagerank\n",
    "    try:\n",
    "        data=nx.pagerank(graph,weight=weight)\n",
    "        for k in graph.nodes(): df.ix[k,'Pagerank Cen'] = data[k]\n",
    "    except Exception as e:\n",
    "        print 'Unable to compute pagerank.'\n",
    "        print 'Error:',e\n",
    "    #Eigenvector centrality\n",
    "    try:\n",
    "        data=nx.eigenvector_centrality_numpy(graph, weight=weight)\n",
    "        for k in graph.nodes(): df.ix[k,'Eigen Cen'] = data[k]\n",
    "    except Exception as e:\n",
    "        print 'Unable to compute eigenvector centrality.'\n",
    "        print 'Error:',e\n",
    "    \n",
    "    return df\n",
    " \n",
    "def compute_graph_statistics_on_pair_of_nodes(graph,weight=None):\n",
    "    df = pd.DataFrame(index=[(x,y) for x in graph.nodes() for y in graph.nodes()])\n",
    "    #Communicability\n",
    "    try:\n",
    "        data=nx.communicability(graph)\n",
    "        for k in graph.nodes():\n",
    "            for v in graph.nodes():\n",
    "                df.ix[(k,v),\"Communicability\"] = data[k][v]\n",
    "    except Exception as e:\n",
    "        print 'Unable to compute communicability.'\n",
    "        print 'Error:',e\n",
    "    #Shortest path\n",
    "    try:\n",
    "        data=nx.shortest_path_length(graph,weight=weight)\n",
    "        for k in graph.nodes():\n",
    "            for v in graph.nodes():\n",
    "                try:\n",
    "                    df.ix[(k,v),\"Shortest path\"] = data[k][v]\n",
    "                except Exception as e:\n",
    "                    print 'error saving...',e\n",
    "                    print data[k][v]\n",
    "    except Exception as e:\n",
    "        print 'Unable to compute shortest path length..'\n",
    "        print 'Error:',e\n",
    "     \n",
    "    return(df)\n",
    "\n",
    "def compute_graph_statistics(graph,stats_dir_name,prefix_file):\n",
    "    data_frame_node = pd.DataFrame()\n",
    "    filename = \"%s/%s-nodes-stats.csv\" % (stats_dir_name,prefix_file)\n",
    "    if (os.path.exists(filename)):\n",
    "        data_frame_node = data_frame_node.from_csv(filename)\n",
    "    else:\n",
    "        data_frame_node = compute_graph_statistics_on_nodes(graph,weight='weight')\n",
    "        data_frame_node.to_csv(filename,encoding=\"utf-8\")\n",
    "    \n",
    "    data_frame_pair = pd.DataFrame()\n",
    "    filename = \"%s/%s-pair-stats.csv\" % (stats_dir_name,prefix_file)\n",
    "    if (os.path.exists(filename)):\n",
    "        data_frame_pair = data_frame_pair.from_csv(filename)\n",
    "    else:\n",
    "        data_frame_pair = compute_graph_statistics_on_pair_of_nodes(graph,weight='distance')\n",
    "        data_frame_pair.to_csv(filename,encoding=\"utf-8\")\n",
    "    \n",
    "    return(data_frame_node,data_frame_pair)\n",
    "\n",
    "\n",
    "## ## ## ## ## ## ## ## ## ## ##\n",
    "\n",
    "if (print_t): print 'Computation of statistics on projected graph page.'\n",
    "df_pro_graph_page_node,df_pro_graph_page_pair = compute_graph_statistics(projected_graph_page,stats_dir_name,'projected_graph_page')\n",
    "  \n",
    "if (print_v): \n",
    "    print '> dataframe of statistics on projected graph page node: df_pro_graph_page_node'\n",
    "    print '> dataframe of statistics on projected graph page pair of nodes: df_pro_graph_page_pair'\n",
    "    \n",
    "\n",
    "if (print_t): print 'Computation of statistics on links graph.'\n",
    "df_links_graph_node,df_links_graph_pair = compute_graph_statistics(links_graph,stats_dir_name,'links_graph')\n",
    "\n",
    "if (print_v): \n",
    "    print '> dataframe of statistics on links graph node: df_links_graph_node'\n",
    "    print '> dataframe of statistics on links graph pair of nodes: df_links_graph_pair'\n",
    "\n",
    "\n",
    "if (print_t): print 'Computation of statistics on occurences graph.'\n",
    "df_occurences_graph_node,df_occurences_graph_pair = compute_graph_statistics(occurences_graph,stats_dir_name,'occurences_graph')\n",
    "\n",
    "if (print_v): \n",
    "    print '> dataframe of statistics on occurences graph node: df_occurences_graph_node'\n",
    "    print '> dataframe of statistics on occurences graph pair of nodes: df_occurences_graph_pair'\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#print len(links_graph.nodes())\n",
    "\n",
    "#res = nx.shortest_path_length(links_graph,weight='distance')\n",
    "#for so in res:\n",
    "#    print len(res[so])\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Intersection graph\n",
    "\n",
    "Build a graph using links graph et projection: a edge between two page not far away in links graph et adjacent in projection "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#def intersection_graph():\n",
    "#    global projected_graph_page, links_graph\n",
    "#    global df_links_graph_pair\n",
    "    \n",
    "#    res = nx.Graph()\n",
    "#    for p in links_graph.nodes():\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# reading map based on a reduced graph of co-edited pages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}