Extract Datasets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-29T15:23:33.937900Z",
     "start_time": "2019-06-29T15:23:32.682488Z"
    }
   },
   "outputs": [],
   "source": [
    "import kaggle\n",
    "\n",
    "\n",
    "kaggle.api.authenticate()\n",
    "list=kaggle.api.dataset_list(file_type=\"csv\",page=1)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#kaggle.api.dataset_download_files('The_name_of_the_dataset', path='the_path_you_want_to_download_the_files_to', unzip=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-29T15:26:48.804883Z",
     "start_time": "2019-06-29T15:26:48.800895Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getlist(inputstring):\n",
    "    inputstring=str(inputstring)\n",
    "    inputstring=inputstring.replace(\"[\",\"\")\n",
    "    inputstring=inputstring.replace(\"]\",\"\")\n",
    "    return inputstring.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-29T15:26:52.067077Z",
     "start_time": "2019-06-29T15:26:51.042213Z"
    }
   },
   "outputs": [],
   "source": [
    "getlist(kaggle.api.dataset_list(file_type=\"csv\",page=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-29T15:35:56.224588Z",
     "start_time": "2019-06-29T15:30:34.768828Z"
    }
   },
   "outputs": [],
   "source": [
    "i=1\n",
    "returnlist=[1]\n",
    "filelist=[]\n",
    "while returnlist != []:\n",
    "    returnlist=getlist(kaggle.api.dataset_list(file_type=\"csv\",page=i))\n",
    "    if i==1:\n",
    "        filelist=returnlist\n",
    "    else:\n",
    "        filelist.append(returnlist)\n",
    "    i+=1\n",
    "    if i % 100==0:\n",
    "        print(i)\n",
    "    \n",
    "filelist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-29T15:50:35.590853Z",
     "start_time": "2019-06-29T15:50:35.573858Z"
    }
   },
   "outputs": [],
   "source": [
    "filelist\n",
    "flat_list = []\n",
    "for sublist in filelist:\n",
    "    if len(sublist[1])==1:\n",
    "        flat_list.append(sublist)\n",
    "        continue\n",
    "    else:\n",
    "        for item in sublist:\n",
    "            flat_list.append(item)\n",
    "flat_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-29T16:00:39.812232Z",
     "start_time": "2019-06-29T16:00:39.795225Z"
    }
   },
   "outputs": [],
   "source": [
    "first5000files=flat_list[0:5000]\n",
    "first5000files = [x.strip() for x in first5000files]\n",
    "first5000files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2019-06-29T16:34:10.814Z"
    }
   },
   "outputs": [],
   "source": [
    "continuefile=\"crawford/us-charities-and-nonprofits\"\n",
    "skip=True\n",
    "for filename in first5000files:\n",
    "    if filename==continuefile:\n",
    "        skip=False\n",
    "    if skip:\n",
    "        continue\n",
    "    print(filename)\n",
    "    try:\n",
    "        file=kaggle.api.dataset_download_files(dataset=filename,unzip=True,path=\"kagglefiles/\"+filename)\n",
    "    except:\n",
    "        print(\"error:\"+filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-29T15:16:58.929864Z",
     "start_time": "2019-06-29T15:16:54.635073Z"
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}