xml_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Element 'data' at 0x1068ce310>\n",
      "2008\n"
     ]
    }
   ],
   "source": [
    "import xml.etree.ElementTree as ET  \n",
    "tree = ET.parse('/Users/joshtitlow/Desktop/items.xml')  \n",
    "root = tree.getroot()\n",
    "print root\n",
    "# one specific item attribute\n",
    "#print('Item #2 attribute:')  \n",
    "print(root[0][1].text)\n",
    "\n",
    "# all item attributes\n",
    "#print('\\nAll attributes:')  \n",
    "#for elem in root:  \n",
    "#    for subelem in elem:\n",
    "#        print(subelem.attrib)\n",
    "\n",
    "# one specific item's data\n",
    "#print('\\nItem #2 data:')  \n",
    "#print(root[0][1].text)\n",
    "\n",
    "# all items data\n",
    "#print('\\nAll item data:')  \n",
    "#for elem in root:  \n",
    "#    print (elem.text)\n",
    "#    for subelem in elem:\n",
    "#        print(subelem.text)\n",
    "\n",
    "#print('\\nAll item names:')  \n",
    "#for elem in root:  \n",
    "#    print ('this is root:',elem.attrib)\n",
    "#    for subelem in elem:\n",
    "#        print(subelem.attrib)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Item #2 data:\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "import xml.etree.ElementTree as ET  \n",
    "tree = ET.parse('matl.omp2info')\n",
    "root = tree.getroot()\n",
    "\n",
    "# all item attributes\n",
    "#print('\\nAll attributes:')  \n",
    "#for elem in root:  \n",
    "#    for subelem in elem:\n",
    "#        print(subelem.attrib)\n",
    "\n",
    "# one specific item's data      \n",
    "print('\\nItem #2 data:')  \n",
    "print(root[0][1].text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "findall() takes at least 2 arguments (1 given)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-39-f0c0c4aefad8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0melem\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mroot\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0msubelem\u001b[0m \u001b[0;32min\u001b[0m \u001b[0melem\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfindall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m         \u001b[0;31m# if we don't need to know the name of the attribute(s), get the dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0;32mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msubelem\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: findall() takes at least 2 arguments (1 given)"
     ]
    }
   ],
   "source": [
    "for elem in root:  \n",
    "    for subelem in elem.findall():\n",
    "\n",
    "        # if we don't need to know the name of the attribute(s), get the dict\n",
    "        print(subelem.text)      \n",
    "\n",
    "        # if we know the name of the attribute, access it directly\n",
    "        #print(subelem.get())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('3733_Dorsal', '20180824_60x_cycle_A01_G001_0001.oir')"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "root[1][4].text, root[1][6][1].text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_dict = {}\n",
    "for group in root.findall('group'):\n",
    "    group.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "old filename = ('20180824_60x_cycle_A01_G001_0001.oir', '20180824_60x_cycle_A01_G010_0001.oir')\n",
      "20180824_CPTI003733_Dorsal.tiff\n",
      "20180824_CPTI002905_Dorsal.tiff\n"
     ]
    }
   ],
   "source": [
    "# Rename Olympus matl experiment files with descriptions\n",
    "# created: 8 September 2018\n",
    "#\n",
    "# Run from inside target directory\n",
    "#\n",
    "# Assumes .oir files have been converted to .tiff\n",
    "#     > if not, then modify input file suffix below\n",
    "#\n",
    "# May need to modify filename output\n",
    "\n",
    "import os\n",
    "import xml.etree.ElementTree as ET\n",
    "\n",
    "tree = ET.parse('matl.omp2info')\n",
    "root = tree.getroot()\n",
    "\n",
    "#get cwd\n",
    "cwd = os.getcwd()\n",
    "\n",
    "# map matl namespace\n",
    "ns = {'matl':'http://www.olympus.co.jp/hpf/protocol/matl/model/matl'}\n",
    "\n",
    "# create a dictionary for image name and description\n",
    "file_dict = {}\n",
    "\n",
    "# add image name and description to the dictionary\n",
    "for group in root.findall('matl:group', ns):\n",
    "    for description in group.findall('matl:description', ns):\n",
    "        filename = description.text\n",
    "    for area in group.findall('matl:area', ns):\n",
    "        for image in area.findall('matl:image', ns):\n",
    "            fileId = image.text\n",
    "        file_dict[fileId]= filename\n",
    "\n",
    "#files = ('20180824_60x_cycle_A01_G001_0001.oir', '20180824_60x_cycle_A01_G010_0001.oir')\n",
    "#print 'old filename =', files\n",
    "\n",
    "# replace image name with description nameL\n",
    "for i in os.listdir(cwd):\n",
    "    if i.endswith('.tiff'):\n",
    "        for fileId, filename in file_dict.items(): \n",
    "            if i in fileId and 'OrR' not in fileId:\n",
    "                newfile = i.replace(i, '20180824_CPTI00'+filename+'.tiff') # input file suffix\n",
    "                os.rename(i, newfile) \n",
    "            if i in fileId and 'OrR' in fileId:\n",
    "                newfile = i.replace(i, '20180824_'+filename+'.tiff') # filename output\n",
    "                os.rename(i, newfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'OrderedDict' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-257-32d207d4268d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;31m# Arguments\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mCPTI_genes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mOrderedDict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m \u001b[0mCPTI_genes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'CPTI001308'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'CG32479'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0mCPTI_genes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'CPTI001586'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'CG31738'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'OrderedDict' is not defined"
     ]
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "jsondata = 'json_data.txt'\n",
    "figure = json.dumps(jsondata)\n",
    "image_name_ID = {}\n",
    "\n",
    "# Arguments\n",
    "CPTI_genes = OrderedDict()\n",
    "CPTI_genes['CPTI001308'] = 'CG32479'\n",
    "CPTI_genes['CPTI001586'] = 'CG31738'\n",
    "CPTI_genes['CPTI001771'] = 'CG31694'\n",
    "CPTI_genes['CPTI003100'] = 'CG32062'\n",
    "\n",
    "# items to replace\n",
    "filename= 'AMP-deaminase_CPTI001081_VNC_venusYFP_zegami1'\n",
    "\n",
    "# function for common replace statements\n",
    "def dataset_replace(data):\n",
    "    data = data.replace('20180710_CPTI_AMPdeaminase', dataset_name) # dataset name\n",
    "    data = data.replace('11402', str(dataset_id)) # dataset ID\n",
    "    data = data.replace('AMP-deaminase', gene) # figure name\n",
    "    data = data.replace('CPTI001081', CPTI) # figure name\n",
    "\n",
    "    # write data to json file\n",
    "    figure_fpath = None\n",
    "    figure_fpath = 'json_data.txt'\n",
    "    #with open('json_data.txt', 'w+b') as filehandle:\n",
    "    figure_json = json.loads(data)\n",
    "        # function for common file export statement\n",
    "\n",
    "#def export(json_data):\n",
    "    # put the string back into json and save a copy of the new json string\n",
    "    #figure_json = json.loads(json_data)\n",
    "\n",
    "#    with open('json_data.txt', 'w+b') as filehandle:\n",
    "#        filehandle.write(m)\n",
    "\n",
    "\n",
    "# David's code to generate OMERO.figures from json text\n",
    "#def main(figure_fpath):\n",
    "    #figure_json = None\n",
    "    #with open(figure_fpath, 'r') as fh:\n",
    "    #    figure_json = json.load(figure_fpath)\n",
    "    if figure_json.has_key('fileId'):\n",
    "        raise RuntimeError('this figure refers an already existing annotation')\n",
    "    name = figure_json['figureName']\n",
    "    description = {'name': name}\n",
    "    file_ann = conn.createFileAnnfromLocalFile(figure_fpath,\n",
    "        origFilePathAndName=name, mimetype='application/json',\n",
    "        ns='omero.web.figure.json', desc=description)\n",
    "\n",
    "def find_replace_name(gene, compartment):\n",
    "    [x for x in image_name_ID if CPTI in x and 'compartment' in x]\n",
    "\n",
    "def find_replace_ID(gene, compartment):\n",
    "    [value for key, value in image_name_ID.items() if gene in key and 'compartment' in key]\n",
    "\n",
    "for CPTI, gene in CPTI_genes.items():\n",
    "\n",
    "    # MB figure\n",
    "    # load template json text and replace filenames\n",
    "    MBtemplate = open('template_json_mb.txt','r')\n",
    "    dataimage_dorsal = find_replace_name(CPTI, 'dorsal')\n",
    "    dataID_dorsal = find_replace_ID(CPTI, 'dorsal')\n",
    "    dataimage_ventral = find_replace_name(CPTI, 'ventral')\n",
    "    dataID_ventral = find_replace_ID(CPTI, 'ventral')\n",
    "    controlimage_dorsal = find_replace_name('OrR', 'dorsal')\n",
    "    controlID_dorsal = find_replace_ID('OrR', 'dorsal')\n",
    "    controlimage_ventral = find_replace_name('OrR', 'ventral')\n",
    "    controlID_ventral = find_replace_ID('OrR', 'ventral')\n",
    "\n",
    "    linesMB = MBtemplate.readlines()\n",
    "    #print(len(lines))\n",
    "    m = linesMB[0]\n",
    "    m = m.replace('CPTI_AMPdeaminase_lobe_dorsal.oif', str(dataimage_dorsal)[3:-2]) # dorsal data\n",
    "    m = m.replace('525320', str(dataID_dorsal)[1:-2]) # dorsal ID\n",
    "    m = m.replace('OrR_CPTI_AMPdeaminase_lobe_dorsal.oif', str(controlimage_dorsal)[3:-2]) # dorsal control\n",
    "    m = m.replace('525322', str(controlID_dorsal)[1:-2]) # dorsal control ID\n",
    "    m = m.replace('CPTI_AMPdeaminase_lobe_ventral.oif', str(dataimage_ventral)[3:-2]) # ventral data\n",
    "    m = m.replace('525316', str(dataID_ventral)[1:-2]) # ventral ID\n",
    "    m = m.replace('OrR_CPTI_AMPdeaminase_lobe_ventral.oif', str(controlimage_ventral)[3:-2]) # ventral control\n",
    "    m = m.replace('525321', str(controlID_ventral)[1:-2]) # ventral control ID\n",
    "    dataset_replace(m)\n",
    "    #print m\n",
    "    #figure_json = json.loads(json_data)\n",
    "    #export(m)\n",
    "    main(json_data)\n",
    "\n",
    "    # overview figure\n",
    "    overviewtemplate= open('template_json_overview.txt')\n",
    "    dataimage_o = find_replace_name(CPTI, 'overview')\n",
    "    dataID_o = find_replace_ID(CPTI, 'overview')\n",
    "    controlimage_o = find_replace_name('OrR', 'overview')\n",
    "    controlID_o = find_replace_ID('OrR', 'overview')\n",
    "\n",
    "    lineso = overviewtemplate.readlines()\n",
    "    m = lineso[0]\n",
    "    m = m.replace('20180709_CPTI_AMPdeaminase_YFP_syp_20x_dorsal_overview.oir', str(dataimage_o)[3:-2]) # overiew data\n",
    "    m = m.replace('525317', str(dataID_o)[1:-2]) # overview ID\n",
    "    dataset_replace(m)\n",
    "    export()\n",
    "    main(json_data)\n",
    "\n",
    "    # CB figure\n",
    "    CBtemplate = open('template_json_cb.txt','r')\n",
    "\n",
    "    linesCB = CBtemplate.readlines()\n",
    "    m = linesCB[0]\n",
    "    m = m.replace('CPTI_AMPdeaminase_lobe_ventral.oif', str(dataimage_ventral)[3:-2]) # ventral data\n",
    "    m = m.replace('525316', str(dataID_ventral)[1:-2]) # ventral ID\n",
    "    m = m.replace('OrR_CPTI_AMPdeaminase_lobe_ventral.oif', str(controlimage_ventral)[3:-2]) # ventral control\n",
    "    m = m.replace('525321', str(controlID_ventral)[1:-2]) # ventral control ID\n",
    "    dataset_replace(m)\n",
    "    export()\n",
    "    main(json_data)\n",
    "\n",
    "    # OL figure\n",
    "    OLtemplate = open('template_json_ol.txt','r')\n",
    "\n",
    "    linesOL = OLtemplate.readlines()\n",
    "    m = linesOL[0]\n",
    "    m = m.replace('CPTI_AMPdeaminase_lobe_ventral.oif', str(dataimage_ventral)[3:-2]) # ventral data\n",
    "    m = m.replace('525316', str(dataID_ventral)[1:-2]) # ventral ID\n",
    "    m = m.replace('OrR_CPTI_AMPdeaminase_lobe_ventral.oif', str(controlimage_ventral)[3:-2]) # ventral control\n",
    "    m = m.replace('525321', str(controlID_ventral)[1:-2]) # ventral control ID\n",
    "    dataset_replace(m)\n",
    "    export()\n",
    "    main(json_data)\n",
    "\n",
    "    # VNC figure\n",
    "    VNCtemplate = open('template_json_VNC.txt','r')\n",
    "    dataimage_VNC = find_replace_name(CPTI, 'VNC')\n",
    "    dataID_VNC = find_replace_ID(CPTI, 'VNC')\n",
    "    controlimage_VNC = find_replace_name('OrR', 'VNC')\n",
    "    controlID_VNC = find_replace_ID('OrR', 'VNC')\n",
    "\n",
    "    linesVNC = VNCtemplate.readlines()\n",
    "    m = linesVNC[0]\n",
    "    m = m.replace('CPTI_AMPdeaminase_VNC_ventral.oif', str(dataimage_VNC)[3:-2]) # VNC data\n",
    "    m = m.replace('525323', str(dataID_ventral)[1:-2]) # VNC ID\n",
    "    m = m.replace('OrR_CPTI_AMPdeaminase_VNC_ventral.oif', str(controlimage_VNC)[3:-2]) # VNC control\n",
    "    m = m.replace('525318', str(controlID_VNC)[1:-2]) # VNC control ID\n",
    "    dataset_replace(m)\n",
    "    export()\n",
    "    main(json_data)\n",
    "\n",
    "conn.close()\n",
    "\n",
    "# TO DO\n",
    "#\n",
    "# Fix mistakes\n",
    "#    > gene names not matched with CPTI numbers\n",
    "#    > gene name didn't get replaced for optic lobe\\\n",
    "\n",
    "# make a new overview template with 4 channels\n",
    "# upload 2nd dataset to OMERO\n",
    "#    > write code to parse the .matl file\n",
    "# make the sequential replace statements more elegant\n",
    "\n",
    "\n",
    "if m.has_key('fileId'):\n",
    "    raise RuntimeError('this figure refers an already existing annotation')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-253-8b9c519ec6b8>, line 8)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-253-8b9c519ec6b8>\"\u001b[0;36m, line \u001b[0;32m8\u001b[0m\n\u001b[0;31m    for i in figure[\"panels\"][][\"datasetId\"]:\u001b[0m\n\u001b[0m                              ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "with open('json_data.txt') as f:\n",
    "    figure = json.load(f)\n",
    "if figure.has_key('fileId'):\n",
    "    raise RuntimeError('this figure refers an already existing annotation')\n",
    "a = figure[\"panels\"][0][\"name\"] \n",
    "#figure\n",
    "b = figure[\"panels\"][0][\"datasetName\"]\n",
    "for i in figure[\"panels\"][][\"datasetId\"]:\n",
    "    print i\n",
    "    #key\n",
    "#a,b, c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2115427,\n",
       " 2115428,\n",
       " 2115429,\n",
       " 2115430,\n",
       " 2115431,\n",
       " 2115432,\n",
       " 2115433,\n",
       " 2115434,\n",
       " 2115435,\n",
       " 2115436,\n",
       " 2115437,\n",
       " 2115438,\n",
       " 2115439,\n",
       " 2115440,\n",
       " 2115441,\n",
       " 2115442,\n",
       " 2115443,\n",
       " 2115444,\n",
       " 2115445,\n",
       " 2115446,\n",
       " 2115447,\n",
       " 2115448,\n",
       " 2115449,\n",
       " 2115450,\n",
       " 2115451,\n",
       " 2115452,\n",
       " 2115453,\n",
       " 2115454,\n",
       " 2115455,\n",
       " 2115456,\n",
       " 2115457,\n",
       " 2115458,\n",
       " 2115459,\n",
       " 2115460,\n",
       " 2115461,\n",
       " 2115462,\n",
       " 2115463,\n",
       " 2115464,\n",
       " 2115465,\n",
       " 2115466,\n",
       " 2115467,\n",
       " 2115468,\n",
       " 2115469,\n",
       " 2115470,\n",
       " 2115471,\n",
       " 2115472]"
      ]
     },
     "execution_count": 260,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "range(2115427, 2115473)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['convertall.py', 'Extract_volume_files.py', 'Find_AND_Sum_Volumes.py', 'FQ_mkdirs.py', 'FQ_outline.py', 'merge.py', 'Omero_batch_import.py', 'pixel_colocalisation.py', 'pixel_colocalisation_v2.py', 'pixel_heatmaps.py', 'rename.py', 'RmvCharactersFileName.py', 'RSS.py', 'RSS_calc.py', 'Sum_multiple_csvFiles_Pandas.py']\n"
     ]
    }
   ],
   "source": [
    "indir = [i for i in os.listdir(cwd) if i.endswith('.py')]\n",
    "print indir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "convertall.py\n",
      "Extract_volume_files.py\n",
      "Find_AND_Sum_Volumes.py\n",
      "FQ_mkdirs.py\n",
      "FQ_outline.py\n",
      "merge.py\n",
      "Omero_batch_import.py\n",
      "pixel_colocalisation.py\n",
      "pixel_colocalisation_v2.py\n",
      "pixel_heatmaps.py\n",
      "rename.py\n",
      "RmvCharactersFileName.py\n",
      "RSS.py\n",
      "RSS_calc.py\n",
      "Sum_multiple_csvFiles_Pandas.py\n"
     ]
    }
   ],
   "source": [
    "for i in os.listdir(cwd):\n",
    "    if i.endswith('.py'):\n",
    "        print i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}