OMERO_extract_imageID_from_json.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    old_imageIds old_json\n",
      "0         404366  2097660\n",
      "10        404360  2097660\n",
      "18        283965  2097660\n",
      "22        404368  2097661\n",
      "26        404364  2097661\n",
      "Empty DataFrame\n",
      "Columns: [missing_json]\n",
      "Index: []\n",
      "    broken_figs\n",
      "0       2091470\n",
      "1       2091472\n",
      "2       2091473\n",
      "3       2091474\n",
      "4       2091484\n",
      "5       2093148\n",
      "6       2093148\n",
      "7       2093150\n",
      "8       2093151\n",
      "9       2093205\n",
      "10      2095519\n",
      "11      2095532\n",
      "12      2095533\n",
      "13      2095534\n",
      "14      2095552\n",
      "15      2095553\n",
      "16      2095556\n",
      "17      2095557\n",
      "18      2095558\n",
      "19      2095560\n",
      "20      2095561\n",
      "21      2095563\n",
      "22      2095564\n",
      "23      2096805\n",
      "24      2096807\n",
      "25      2096811\n",
      "26      2096814\n",
      "27      2097264\n",
      "28      2097411\n",
      "29      2097449\n",
      "..          ...\n",
      "230            \n",
      "231            \n",
      "232            \n",
      "233            \n",
      "234            \n",
      "235            \n",
      "236            \n",
      "237            \n",
      "238            \n",
      "239            \n",
      "240            \n",
      "241            \n",
      "242            \n",
      "243            \n",
      "244            \n",
      "245            \n",
      "246            \n",
      "247            \n",
      "248            \n",
      "249            \n",
      "250            \n",
      "251            \n",
      "252            \n",
      "253            \n",
      "254            \n",
      "255            \n",
      "256            \n",
      "257            \n",
      "258            \n",
      "259            \n",
      "\n",
      "[260 rows x 1 columns]\n"
     ]
    }
   ],
   "source": [
    "#####################################################################################\n",
    "# Extract imageID from OMEROfigure json file\n",
    "#\n",
    "# --- Requires list of json IDs and/or directory of json files\n",
    "#\n",
    "# -\n",
    "\n",
    "import pandas as pd\n",
    "import json\n",
    "import os\n",
    "\n",
    "# get json files\n",
    "indir = '/Volumes/bioc1301/src/Zegami_scripts/Zegami_collection_27_Feb_2019/figures/'\n",
    "outdir = '/Volumes/bioc1301/src/OMERO_scripts/OMERO_Figure_screen_dataset/'\n",
    "\n",
    "#json_files = [indir+file for file in os.listdir(indir) if file.endswith('.json')]\n",
    "figure_IDs = '/Volumes/bioc1301/src/OMERO_scripts/OMERO_Figure_screen_dataset/CNS_figureIDs.csv'\n",
    "figure_IDs = open(figure_IDs).read().splitlines()\n",
    "\n",
    "# instantiate a list to store image filenames and broken figures\n",
    "image_list = []\n",
    "broken_figures = []\n",
    "json_list = []\n",
    "missing_json = []\n",
    "    \n",
    "for figID in figure_IDs:\n",
    "    try:\n",
    "        jsonfile = os.path.join(indir, figID+\".json\")\n",
    "    except:\n",
    "        missing_json.append(figID) \n",
    "        pass\n",
    "    try:\n",
    "        with open(jsonfile) as datafile:  \n",
    "            data = json.load(datafile)\n",
    "            for i in data['panels']:\n",
    "                image_list.append(i['imageId'])\n",
    "                json_list.append(figID)\n",
    "    except:\n",
    "        broken_figures.append(figID)\n",
    "        pass\n",
    "\n",
    "df = pd.DataFrame({'old_imageIds':image_list, 'old_json':json_list})\n",
    "df = df[['old_imageIds', 'old_json']]\n",
    "df = df.drop_duplicates(['old_imageIds'])\n",
    "print df.head()\n",
    "df.to_csv(os.path.join(outdir,'CNS_imageID_json.csv'), index=True)\n",
    "df1 = pd.DataFrame({'missing_json':missing_json})\n",
    "df2 = pd.DataFrame({'broken_figs':broken_figures})\n",
    "df1 = df1[['missing_json']]\n",
    "df2 = df2[['broken_figs']]\n",
    "print df1\n",
    "print df2\n",
    "#errors = errors(['missing_json', 'broken_figs'])\n",
    "#errors.to_csv(os.path.join(indir,'errors.csv'), index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#with open(\"broken_figures.txt\", \"w\") as output:\n",
    "#    output.write(str(broken_figures))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    2110855\n",
       "1    2119346\n",
       "Name: broken_figs, dtype: object"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = pd.DataFrame['/Volumes/bioc1301/src/OMERO_scripts/test/NMJ_figureIDs.csv']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2.to_csv(os.path.join(outdir,'CNS_missing_jsons.csv'), index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "ename": "IOError",
     "evalue": "[Errno 2] No such file or directory: '/Volumes/bioc1301/src/Zegami_scripts/Zegami_collection_27_Feb_2019/figures/2097638.json'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIOError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-96b6d0f9188f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mjsonfile\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'/Volumes/bioc1301/src/Zegami_scripts/Zegami_collection_27_Feb_2019/figures/2097638.json'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjsonfile\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mdatafile\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdatafile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'panels'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIOError\u001b[0m: [Errno 2] No such file or directory: '/Volumes/bioc1301/src/Zegami_scripts/Zegami_collection_27_Feb_2019/figures/2097638.json'"
     ]
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "jsonfile = '/Volumes/bioc1301/src/Zegami_scripts/Zegami_collection_27_Feb_2019/figures/2097638.json'\n",
    "with open(jsonfile) as datafile:  \n",
    "        data = json.load(datafile)\n",
    "        for i in data['panels']:\n",
    "            print i['imageId']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'/Volumes/bioc1301/src/OMERO_scripts'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:anaconda]",
   "language": "python",
   "name": "conda-env-anaconda-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}