alphapept_script_dev.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "import pandas as pd\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import os\n",
    "import itertools\n",
    "import gc\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "from mimas.tools.spectral_file.extract_ms1_feature import process_mzml_file\n",
    "import time\n",
    "import fnmatch\n",
    "import os\n",
    "import sys\n",
    "def find_files(base, pattern):\n",
    "    '''Return list of files matching pattern in base folder.'''\n",
    "    return [filename for filename in os.listdir(base) if re.search(pattern, filename, re.IGNORECASE)]\n",
    "mzml_dir = \"/Volumes/Samsung_T5/MZMLs\"\n",
    "source_dir = \"/Users/fanzhoukong/Documents/GitHub/Libgen_data/GT\""
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "ms2_selected = process_mzml_file(os.path.join(mzml_dir, \"2A1_Mix_5.mzML\"))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n",
      "i am processing mix 1, out of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/109 [00:02<?, ?it/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m                         Traceback (most recent call last)",
      "Input \u001B[0;32mIn [188]\u001B[0m, in \u001B[0;36m<cell line: 34>\u001B[0;34m()\u001B[0m\n\u001B[1;32m     48\u001B[0m             missing_files\u001B[38;5;241m.\u001B[39mappend(filename)\n\u001B[1;32m     49\u001B[0m         \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mlen\u001B[39m(found_file)\u001B[38;5;241m==\u001B[39m\u001B[38;5;241m1\u001B[39m:\n\u001B[0;32m---> 50\u001B[0m             ms2_selected \u001B[38;5;241m=\u001B[39m \u001B[43mprocess_mzml_file\u001B[49m\u001B[43m(\u001B[49m\u001B[43mos\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mpath\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mjoin\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmzml_dir\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfound_file\u001B[49m\u001B[43m[\u001B[49m\u001B[38;5;241;43m0\u001B[39;49m\u001B[43m]\u001B[49m\u001B[43m)\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m     51\u001B[0m             ms2_selected\u001B[38;5;241m.\u001B[39mto_csv((os\u001B[38;5;241m.\u001B[39mpath\u001B[38;5;241m.\u001B[39mjoin(output_folder_path, filename\u001B[38;5;241m+\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m.csv\u001B[39m\u001B[38;5;124m\"\u001B[39m)), index\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mFalse\u001B[39;00m)\n\u001B[1;32m     52\u001B[0m \u001B[38;5;28mprint\u001B[39m(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n",
      "File \u001B[0;32m~/Documents/GitHub/Libgen_2.0/mimas/tools/spectral_file/extract_ms1_feature.py:30\u001B[0m, in \u001B[0;36mprocess_mzml_file\u001B[0;34m(file_input, ifSciex, ifdebug)\u001B[0m\n\u001B[1;32m     22\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mprocess_mzml_file\u001B[39m(file_input, ifSciex \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mTrue\u001B[39;00m, ifdebug \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mFalse\u001B[39;00m):\n\u001B[1;32m     23\u001B[0m     \u001B[38;5;66;03m# path_output.mkdir(parents=True, exist_ok=True)\u001B[39;00m\n\u001B[1;32m     24\u001B[0m     \u001B[38;5;66;03m# file_features_all = path_output / \"features_all.csv\"\u001B[39;00m\n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m     27\u001B[0m \n\u001B[1;32m     28\u001B[0m     \u001B[38;5;66;03m# Read mzML file\u001B[39;00m\n\u001B[1;32m     29\u001B[0m     \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m---> 30\u001B[0m         ms_file \u001B[38;5;241m=\u001B[39m \u001B[43mload_mzml_data\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mstr\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43mfile_input\u001B[49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mifSciex\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mifSciex\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m     31\u001B[0m     \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mException\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m     32\u001B[0m         \u001B[38;5;66;03m# traceback.print_exc(file=open(path_output / \"error_in_reading_mzml.txt\", \"w\"))\u001B[39;00m\n\u001B[1;32m     33\u001B[0m         \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;241m0\u001B[39m\n",
      "File \u001B[0;32m~/Documents/GitHub/Libgen_2.0/mimas/external/features_by_alphapept/load_mzml_data.py:32\u001B[0m, in \u001B[0;36mload_mzml_data\u001B[0;34m(file, n_most_abundant, nested_array, ifSciex)\u001B[0m\n\u001B[1;32m     29\u001B[0m select_windows_list \u001B[38;5;241m=\u001B[39m []\n\u001B[1;32m     31\u001B[0m \u001B[38;5;28mid\u001B[39m \u001B[38;5;241m=\u001B[39m \u001B[38;5;241m1\u001B[39m\n\u001B[0;32m---> 32\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m spec \u001B[38;5;129;01min\u001B[39;00m pymzml\u001B[38;5;241m.\u001B[39mrun\u001B[38;5;241m.\u001B[39mReader(file, obo_version\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m4.1.33\u001B[39m\u001B[38;5;124m\"\u001B[39m):\n\u001B[1;32m     33\u001B[0m     rt, masses, intensities, ms_order, prec_mass, mono_mz, charge, (prec_windows_lower, prec_windows_upper) \u001B[38;5;241m=\u001B[39m _extract_mzml_info(spec)\n\u001B[1;32m     34\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m ifSciex \u001B[38;5;241m==\u001B[39m \u001B[38;5;28;01mTrue\u001B[39;00m:\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/alphapept_env/lib/python3.8/site-packages/pymzml/run.py:153\u001B[0m, in \u001B[0;36mReader.__next__\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m    151\u001B[0m has_ref_group \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39minfo\u001B[38;5;241m.\u001B[39mget(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mreferenceable_param_group_list\u001B[39m\u001B[38;5;124m\"\u001B[39m, \u001B[38;5;28;01mFalse\u001B[39;00m)\n\u001B[1;32m    152\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m:\n\u001B[0;32m--> 153\u001B[0m     event, element \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mnext\u001B[39;49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43miter\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mEND\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mEND\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m)\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    154\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m event \u001B[38;5;241m==\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mend\u001B[39m\u001B[38;5;124m\"\u001B[39m:\n\u001B[1;32m    155\u001B[0m         \u001B[38;5;28;01mif\u001B[39;00m element\u001B[38;5;241m.\u001B[39mtag\u001B[38;5;241m.\u001B[39mendswith(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m}spectrum\u001B[39m\u001B[38;5;124m\"\u001B[39m):\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/alphapept_env/lib/python3.8/xml/etree/ElementTree.py:1232\u001B[0m, in \u001B[0;36miterparse.<locals>.iterator\u001B[0;34m()\u001B[0m\n\u001B[1;32m   1230\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m data:\n\u001B[1;32m   1231\u001B[0m         \u001B[38;5;28;01mbreak\u001B[39;00m\n\u001B[0;32m-> 1232\u001B[0m     \u001B[43mpullparser\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mfeed\u001B[49m\u001B[43m(\u001B[49m\u001B[43mdata\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1233\u001B[0m root \u001B[38;5;241m=\u001B[39m pullparser\u001B[38;5;241m.\u001B[39m_close_and_return_root()\n\u001B[1;32m   1234\u001B[0m \u001B[38;5;28;01myield from\u001B[39;00m pullparser\u001B[38;5;241m.\u001B[39mread_events()\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/alphapept_env/lib/python3.8/xml/etree/ElementTree.py:1274\u001B[0m, in \u001B[0;36mXMLPullParser.feed\u001B[0;34m(self, data)\u001B[0m\n\u001B[1;32m   1272\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m data:\n\u001B[1;32m   1273\u001B[0m     \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m-> 1274\u001B[0m         \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_parser\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mfeed\u001B[49m\u001B[43m(\u001B[49m\u001B[43mdata\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1275\u001B[0m     \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mSyntaxError\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m exc:\n\u001B[1;32m   1276\u001B[0m         \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_events_queue\u001B[38;5;241m.\u001B[39mappend(exc)\n",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m: "
     ]
    }
   ],
   "source": [
    "\n",
    "std_list = pd.read_csv(os.path.join(source_dir, \"std_list.csv\"))\n",
    "std_list = std_list.applymap(lambda x: x.strip() if isinstance(x, str) else x)\n",
    "enzyme_list = pd.read_csv(os.path.join(source_dir, \"enzyme_list.csv\"))\n",
    "enzyme_list = enzyme_list.applymap(lambda x: x.strip() if isinstance(x, str) else x)\n",
    "enzyme_list['List of enzymes']=enzyme_list['List of enzymes'].replace('_', '', regex=True)\n",
    "for index, row in enzyme_list.iterrows():\n",
    "    if row['List of enzymes'].startswith(str(1))==True:\n",
    "        row['List of enzymes']=(row['List of enzymes'][1:])\n",
    "output_folder_path = os.path.join(source_dir, \"features\")\n",
    "if os.path.exists(output_folder_path) == False:\n",
    "        os.mkdir(output_folder_path)\n",
    "missing_files = []\n",
    "duplicated_files = []\n",
    "for mix in std_list['Mix'].unique():\n",
    "    print(\"i am processing mix %s, out of %s\" %(str(mix), str(len(std_list['Mix'].unique()))))\n",
    "    for enzyme in tqdm(enzyme_list['List of enzymes'].unique()):\n",
    "\n",
    "        filename =enzyme+\"_\"+\"Mix\"+\"_\"+str(mix)\n",
    "        found_file = find_files(mzml_dir, f\"^{filename}.mzML\")\n",
    "        if len(found_file)==0:\n",
    "            found_file = find_files(mzml_dir, f\"^{filename}(.*).mzML$\")\n",
    "\n",
    "        if len(found_file)>1:\n",
    "            duplicated_files.append(filename)\n",
    "            # print(filename)\n",
    "        if len(found_file)==0:\n",
    "            # print(\"the file %s doesnt exist\" %filename)\n",
    "            missing_files.append(filename)\n",
    "        if len(found_file)==1:\n",
    "            ms2_selected = process_mzml_file(os.path.join(mzml_dir, found_file[0]))\n",
    "            ms2_selected.to_csv((os.path.join(output_folder_path, filename+\".csv\")), index=False)\n",
    "print(\"there are %s of files duplicated\" % (str(len(duplicated_files))))\n",
    "print(\"there are %s of files missing\" % (str(len(missing_files))))\n",
    "missing_files = pd.DataFrame(missing_files)\n",
    "missing_files.to_csv(os.path.join(source_dir, \"missing_files\"+\".csv\", index = False))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "alphapept_env",
   "language": "python",
   "display_name": "alphapept_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}