Methylation_data_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c99729cd-a9ee-4601-8a46-98035f87e363",
   "metadata": {},
   "outputs": [],
   "source": [
    "import importlib\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import pwd\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "from tqdm import tqdm, trange\n",
    "from tqdm.contrib.concurrent import process_map\n",
    "from meth import tools as mtools\n",
    "import matplotlib.pyplot as plt\n",
    "import GEOparse\n",
    "import scanpy as sc\n",
    "from scipy import sparse as sp\n",
    "import pyreadr\n",
    "\n",
    "# %autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca29873b-f497-4681-bee4-f87f54ecdb9c",
   "metadata": {},
   "source": [
    "# Wayne"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "568918cd-a8c2-4dc1-a336-607382b88f5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "cwd = Path.cwd()\n",
    "project_dir = Path('data/Wayne/')\n",
    "project_id = 'Wayne'\n",
    "project_samples_dir = 'detection_p_values_sesame'\n",
    "path_samples = Path.joinpath(project_dir, project_samples_dir)\n",
    "gl_path_samples = Path.joinpath(cwd, path_samples)\n",
    "gl_project_dir = Path.joinpath(cwd, project_dir)\n",
    "metainfo_path = 'mammalianarray_default_predictions_copy.csv'\n",
    "\n",
    "data_path = 'mammalianarray_default_predictions_copy.csv'\n",
    "gl_metainfo_path = Path.joinpath(project_dir, metainfo_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "aab79c52-35d1-4500-9d88-e22a87caf214",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(330948, 144)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ET0215_AM10L</th>\n",
       "      <th>ET0215_EM3L</th>\n",
       "      <th>ET0215_VF6L</th>\n",
       "      <th>ET0215_RM6L</th>\n",
       "      <th>ET0215_VF2L</th>\n",
       "      <th>ET0215_EF4L</th>\n",
       "      <th>ET0215_ControlF7L</th>\n",
       "      <th>ET0215_AF7L</th>\n",
       "      <th>ET0215_AM7L</th>\n",
       "      <th>ET0215_SM7L</th>\n",
       "      <th>...</th>\n",
       "      <th>ET0215_CF13B</th>\n",
       "      <th>ET0215_AM5B</th>\n",
       "      <th>ET0215_AF9B</th>\n",
       "      <th>ET0215_SF10B</th>\n",
       "      <th>ET0215_RM1B</th>\n",
       "      <th>ET0215_AF3B</th>\n",
       "      <th>ET0215_VM8B</th>\n",
       "      <th>ET0215_CM8B</th>\n",
       "      <th>ET0215_RM5B</th>\n",
       "      <th>ET0215_ControlM6B</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cg00000165_TC21</th>\n",
       "      <td>0.488497</td>\n",
       "      <td>0.480638</td>\n",
       "      <td>0.480988</td>\n",
       "      <td>0.485970</td>\n",
       "      <td>0.453424</td>\n",
       "      <td>0.489981</td>\n",
       "      <td>0.470748</td>\n",
       "      <td>0.463907</td>\n",
       "      <td>0.520466</td>\n",
       "      <td>0.492263</td>\n",
       "      <td>...</td>\n",
       "      <td>0.508456</td>\n",
       "      <td>0.457821</td>\n",
       "      <td>0.489105</td>\n",
       "      <td>0.512363</td>\n",
       "      <td>0.454725</td>\n",
       "      <td>0.492842</td>\n",
       "      <td>0.451083</td>\n",
       "      <td>0.470395</td>\n",
       "      <td>0.524883</td>\n",
       "      <td>0.444731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00001209_TC21</th>\n",
       "      <td>0.843054</td>\n",
       "      <td>0.839191</td>\n",
       "      <td>0.846582</td>\n",
       "      <td>0.854151</td>\n",
       "      <td>0.878952</td>\n",
       "      <td>0.837668</td>\n",
       "      <td>0.877271</td>\n",
       "      <td>0.876795</td>\n",
       "      <td>0.771787</td>\n",
       "      <td>0.890039</td>\n",
       "      <td>...</td>\n",
       "      <td>0.921959</td>\n",
       "      <td>0.899398</td>\n",
       "      <td>0.895369</td>\n",
       "      <td>0.890514</td>\n",
       "      <td>0.887780</td>\n",
       "      <td>0.902595</td>\n",
       "      <td>0.758381</td>\n",
       "      <td>0.860072</td>\n",
       "      <td>0.896445</td>\n",
       "      <td>0.836463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00001364_TC21</th>\n",
       "      <td>0.894383</td>\n",
       "      <td>0.914747</td>\n",
       "      <td>0.913515</td>\n",
       "      <td>0.924633</td>\n",
       "      <td>0.932361</td>\n",
       "      <td>0.902978</td>\n",
       "      <td>0.922835</td>\n",
       "      <td>0.914902</td>\n",
       "      <td>0.837230</td>\n",
       "      <td>0.939791</td>\n",
       "      <td>...</td>\n",
       "      <td>0.894584</td>\n",
       "      <td>0.870177</td>\n",
       "      <td>0.848844</td>\n",
       "      <td>0.792075</td>\n",
       "      <td>0.875768</td>\n",
       "      <td>0.896836</td>\n",
       "      <td>0.801838</td>\n",
       "      <td>0.877272</td>\n",
       "      <td>0.848116</td>\n",
       "      <td>0.873201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00001582_TC21</th>\n",
       "      <td>0.046435</td>\n",
       "      <td>0.043950</td>\n",
       "      <td>0.042922</td>\n",
       "      <td>0.043529</td>\n",
       "      <td>0.035833</td>\n",
       "      <td>0.034553</td>\n",
       "      <td>0.037163</td>\n",
       "      <td>0.036471</td>\n",
       "      <td>0.055088</td>\n",
       "      <td>0.054627</td>\n",
       "      <td>...</td>\n",
       "      <td>0.040198</td>\n",
       "      <td>0.039016</td>\n",
       "      <td>0.047113</td>\n",
       "      <td>0.049125</td>\n",
       "      <td>0.043920</td>\n",
       "      <td>0.041857</td>\n",
       "      <td>0.050875</td>\n",
       "      <td>0.054214</td>\n",
       "      <td>0.049643</td>\n",
       "      <td>0.032139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00002920_TC21</th>\n",
       "      <td>0.682785</td>\n",
       "      <td>0.703191</td>\n",
       "      <td>0.694720</td>\n",
       "      <td>0.731987</td>\n",
       "      <td>0.747875</td>\n",
       "      <td>0.668616</td>\n",
       "      <td>0.777228</td>\n",
       "      <td>0.706941</td>\n",
       "      <td>0.636992</td>\n",
       "      <td>0.790614</td>\n",
       "      <td>...</td>\n",
       "      <td>0.143896</td>\n",
       "      <td>0.184177</td>\n",
       "      <td>0.167003</td>\n",
       "      <td>0.201880</td>\n",
       "      <td>0.181908</td>\n",
       "      <td>0.220735</td>\n",
       "      <td>0.223249</td>\n",
       "      <td>0.210300</td>\n",
       "      <td>0.213973</td>\n",
       "      <td>0.205717</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 144 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 ET0215_AM10L  ET0215_EM3L  ET0215_VF6L  ET0215_RM6L  \\\n",
       "cg00000165_TC21      0.488497     0.480638     0.480988     0.485970   \n",
       "cg00001209_TC21      0.843054     0.839191     0.846582     0.854151   \n",
       "cg00001364_TC21      0.894383     0.914747     0.913515     0.924633   \n",
       "cg00001582_TC21      0.046435     0.043950     0.042922     0.043529   \n",
       "cg00002920_TC21      0.682785     0.703191     0.694720     0.731987   \n",
       "\n",
       "                 ET0215_VF2L  ET0215_EF4L  ET0215_ControlF7L  ET0215_AF7L  \\\n",
       "cg00000165_TC21     0.453424     0.489981           0.470748     0.463907   \n",
       "cg00001209_TC21     0.878952     0.837668           0.877271     0.876795   \n",
       "cg00001364_TC21     0.932361     0.902978           0.922835     0.914902   \n",
       "cg00001582_TC21     0.035833     0.034553           0.037163     0.036471   \n",
       "cg00002920_TC21     0.747875     0.668616           0.777228     0.706941   \n",
       "\n",
       "                 ET0215_AM7L  ET0215_SM7L  ...  ET0215_CF13B  ET0215_AM5B  \\\n",
       "cg00000165_TC21     0.520466     0.492263  ...      0.508456     0.457821   \n",
       "cg00001209_TC21     0.771787     0.890039  ...      0.921959     0.899398   \n",
       "cg00001364_TC21     0.837230     0.939791  ...      0.894584     0.870177   \n",
       "cg00001582_TC21     0.055088     0.054627  ...      0.040198     0.039016   \n",
       "cg00002920_TC21     0.636992     0.790614  ...      0.143896     0.184177   \n",
       "\n",
       "                 ET0215_AF9B  ET0215_SF10B  ET0215_RM1B  ET0215_AF3B  \\\n",
       "cg00000165_TC21     0.489105      0.512363     0.454725     0.492842   \n",
       "cg00001209_TC21     0.895369      0.890514     0.887780     0.902595   \n",
       "cg00001364_TC21     0.848844      0.792075     0.875768     0.896836   \n",
       "cg00001582_TC21     0.047113      0.049125     0.043920     0.041857   \n",
       "cg00002920_TC21     0.167003      0.201880     0.181908     0.220735   \n",
       "\n",
       "                 ET0215_VM8B  ET0215_CM8B  ET0215_RM5B  ET0215_ControlM6B  \n",
       "cg00000165_TC21     0.451083     0.470395     0.524883           0.444731  \n",
       "cg00001209_TC21     0.758381     0.860072     0.896445           0.836463  \n",
       "cg00001364_TC21     0.801838     0.877272     0.848116           0.873201  \n",
       "cg00001582_TC21     0.050875     0.054214     0.049643           0.032139  \n",
       "cg00002920_TC21     0.223249     0.210300     0.213973           0.205717  \n",
       "\n",
       "[5 rows x 144 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# rdata_sesame = pyreadr.read_r(Path.joinpath(project_dir, 'detection_p_values_sesame/all_probes_sesame_normalized.RDS'))\n",
    "df_sesame = pd.read_csv(Path.joinpath(project_dir, 'betas.csv'),index_col=0)\n",
    "# df_sesame.set_index('CGid', inplace=True)\n",
    "print(df_sesame.shape)\n",
    "df_sesame.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "bb834016-6d8d-42d5-a6b3-b517ee6962cf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(330948, 144)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ET0215_AM10L</th>\n",
       "      <th>ET0215_EM3L</th>\n",
       "      <th>ET0215_VF6L</th>\n",
       "      <th>ET0215_RM6L</th>\n",
       "      <th>ET0215_VF2L</th>\n",
       "      <th>ET0215_EF4L</th>\n",
       "      <th>ET0215_ControlF7L</th>\n",
       "      <th>ET0215_AF7L</th>\n",
       "      <th>ET0215_AM7L</th>\n",
       "      <th>ET0215_SM7L</th>\n",
       "      <th>...</th>\n",
       "      <th>ET0215_CF13B</th>\n",
       "      <th>ET0215_AM5B</th>\n",
       "      <th>ET0215_AF9B</th>\n",
       "      <th>ET0215_SF10B</th>\n",
       "      <th>ET0215_RM1B</th>\n",
       "      <th>ET0215_AF3B</th>\n",
       "      <th>ET0215_VM8B</th>\n",
       "      <th>ET0215_CM8B</th>\n",
       "      <th>ET0215_RM5B</th>\n",
       "      <th>ET0215_ControlM6B</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cg00000165_TC21</th>\n",
       "      <td>0.891631</td>\n",
       "      <td>0.734009</td>\n",
       "      <td>0.933971</td>\n",
       "      <td>0.904303</td>\n",
       "      <td>0.795108</td>\n",
       "      <td>0.876646</td>\n",
       "      <td>0.946862</td>\n",
       "      <td>0.903118</td>\n",
       "      <td>0.987531</td>\n",
       "      <td>0.953777</td>\n",
       "      <td>...</td>\n",
       "      <td>0.773402</td>\n",
       "      <td>0.791444</td>\n",
       "      <td>0.973858</td>\n",
       "      <td>0.769434</td>\n",
       "      <td>0.641535</td>\n",
       "      <td>0.926698</td>\n",
       "      <td>0.790507</td>\n",
       "      <td>0.888243</td>\n",
       "      <td>0.759482</td>\n",
       "      <td>0.618919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00001209_TC21</th>\n",
       "      <td>0.009618</td>\n",
       "      <td>0.012510</td>\n",
       "      <td>0.019497</td>\n",
       "      <td>0.015152</td>\n",
       "      <td>0.011236</td>\n",
       "      <td>0.011383</td>\n",
       "      <td>0.010756</td>\n",
       "      <td>0.011999</td>\n",
       "      <td>0.041604</td>\n",
       "      <td>0.006788</td>\n",
       "      <td>...</td>\n",
       "      <td>0.005274</td>\n",
       "      <td>0.010975</td>\n",
       "      <td>0.007665</td>\n",
       "      <td>0.014087</td>\n",
       "      <td>0.010766</td>\n",
       "      <td>0.008553</td>\n",
       "      <td>0.025042</td>\n",
       "      <td>0.015466</td>\n",
       "      <td>0.012291</td>\n",
       "      <td>0.022160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00001364_TC21</th>\n",
       "      <td>0.001692</td>\n",
       "      <td>0.002339</td>\n",
       "      <td>0.002318</td>\n",
       "      <td>0.002193</td>\n",
       "      <td>0.002141</td>\n",
       "      <td>0.001671</td>\n",
       "      <td>0.001984</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>0.002632</td>\n",
       "      <td>0.001629</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001608</td>\n",
       "      <td>0.002141</td>\n",
       "      <td>0.001431</td>\n",
       "      <td>0.002444</td>\n",
       "      <td>0.001796</td>\n",
       "      <td>0.002057</td>\n",
       "      <td>0.001911</td>\n",
       "      <td>0.001577</td>\n",
       "      <td>0.001577</td>\n",
       "      <td>0.001556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00001582_TC21</th>\n",
       "      <td>0.002369</td>\n",
       "      <td>0.002975</td>\n",
       "      <td>0.002231</td>\n",
       "      <td>0.002589</td>\n",
       "      <td>0.003003</td>\n",
       "      <td>0.002617</td>\n",
       "      <td>0.003223</td>\n",
       "      <td>0.002920</td>\n",
       "      <td>0.002314</td>\n",
       "      <td>0.002782</td>\n",
       "      <td>...</td>\n",
       "      <td>0.002700</td>\n",
       "      <td>0.002452</td>\n",
       "      <td>0.002314</td>\n",
       "      <td>0.002507</td>\n",
       "      <td>0.002479</td>\n",
       "      <td>0.002865</td>\n",
       "      <td>0.002066</td>\n",
       "      <td>0.001873</td>\n",
       "      <td>0.002038</td>\n",
       "      <td>0.002314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00002920_TC21</th>\n",
       "      <td>0.033365</td>\n",
       "      <td>0.042147</td>\n",
       "      <td>0.080420</td>\n",
       "      <td>0.047306</td>\n",
       "      <td>0.053102</td>\n",
       "      <td>0.047107</td>\n",
       "      <td>0.031589</td>\n",
       "      <td>0.049948</td>\n",
       "      <td>0.175710</td>\n",
       "      <td>0.023005</td>\n",
       "      <td>...</td>\n",
       "      <td>0.071732</td>\n",
       "      <td>0.165253</td>\n",
       "      <td>0.125558</td>\n",
       "      <td>0.166162</td>\n",
       "      <td>0.132555</td>\n",
       "      <td>0.181973</td>\n",
       "      <td>0.170431</td>\n",
       "      <td>0.199879</td>\n",
       "      <td>0.205746</td>\n",
       "      <td>0.178255</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 144 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 ET0215_AM10L  ET0215_EM3L  ET0215_VF6L  ET0215_RM6L  \\\n",
       "cg00000165_TC21      0.891631     0.734009     0.933971     0.904303   \n",
       "cg00001209_TC21      0.009618     0.012510     0.019497     0.015152   \n",
       "cg00001364_TC21      0.001692     0.002339     0.002318     0.002193   \n",
       "cg00001582_TC21      0.002369     0.002975     0.002231     0.002589   \n",
       "cg00002920_TC21      0.033365     0.042147     0.080420     0.047306   \n",
       "\n",
       "                 ET0215_VF2L  ET0215_EF4L  ET0215_ControlF7L  ET0215_AF7L  \\\n",
       "cg00000165_TC21     0.795108     0.876646           0.946862     0.903118   \n",
       "cg00001209_TC21     0.011236     0.011383           0.010756     0.011999   \n",
       "cg00001364_TC21     0.002141     0.001671           0.001984     0.001848   \n",
       "cg00001582_TC21     0.003003     0.002617           0.003223     0.002920   \n",
       "cg00002920_TC21     0.053102     0.047107           0.031589     0.049948   \n",
       "\n",
       "                 ET0215_AM7L  ET0215_SM7L  ...  ET0215_CF13B  ET0215_AM5B  \\\n",
       "cg00000165_TC21     0.987531     0.953777  ...      0.773402     0.791444   \n",
       "cg00001209_TC21     0.041604     0.006788  ...      0.005274     0.010975   \n",
       "cg00001364_TC21     0.002632     0.001629  ...      0.001608     0.002141   \n",
       "cg00001582_TC21     0.002314     0.002782  ...      0.002700     0.002452   \n",
       "cg00002920_TC21     0.175710     0.023005  ...      0.071732     0.165253   \n",
       "\n",
       "                 ET0215_AF9B  ET0215_SF10B  ET0215_RM1B  ET0215_AF3B  \\\n",
       "cg00000165_TC21     0.973858      0.769434     0.641535     0.926698   \n",
       "cg00001209_TC21     0.007665      0.014087     0.010766     0.008553   \n",
       "cg00001364_TC21     0.001431      0.002444     0.001796     0.002057   \n",
       "cg00001582_TC21     0.002314      0.002507     0.002479     0.002865   \n",
       "cg00002920_TC21     0.125558      0.166162     0.132555     0.181973   \n",
       "\n",
       "                 ET0215_VM8B  ET0215_CM8B  ET0215_RM5B  ET0215_ControlM6B  \n",
       "cg00000165_TC21     0.790507     0.888243     0.759482           0.618919  \n",
       "cg00001209_TC21     0.025042     0.015466     0.012291           0.022160  \n",
       "cg00001364_TC21     0.001911     0.001577     0.001577           0.001556  \n",
       "cg00001582_TC21     0.002066     0.001873     0.002038           0.002314  \n",
       "cg00002920_TC21     0.170431     0.199879     0.205746           0.178255  \n",
       "\n",
       "[5 rows x 144 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_p_sesame = pd.read_csv(Path.joinpath(project_dir, 'pvals.csv'),index_col=0)\n",
    "# df_p_sesame.set_index('CGid', inplace=True)\n",
    "print(df_p_sesame.shape)\n",
    "x = df_p_sesame.values\n",
    "# print(np.sum(np.sum(1.*(x < 0.01), axis=1) >= 16))# x.shape[1]))\n",
    "df_p_sesame.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "a18937e4-1d8d-4103-a3f0-d0d1974ba5e9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(144, 91)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PredictedSpeciesLatinName</th>\n",
       "      <th>ExternalSampleID</th>\n",
       "      <th>Well_Address</th>\n",
       "      <th>Plate_Number</th>\n",
       "      <th>Sample_Group</th>\n",
       "      <th>Pool_ID</th>\n",
       "      <th>Sentrix_ID</th>\n",
       "      <th>Sentrix_Position</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Species</th>\n",
       "      <th>...</th>\n",
       "      <th>LifespanUberClock_Heart</th>\n",
       "      <th>LifespanUberClock_Kidney</th>\n",
       "      <th>LifespanUberClock_Muscle</th>\n",
       "      <th>LifespanUberClock_Cortex</th>\n",
       "      <th>LifespanUberClock_Striatum</th>\n",
       "      <th>LifespanUberClock_Cerebellum</th>\n",
       "      <th>LifespanUberClock_Brain</th>\n",
       "      <th>LifespanUberClock_Fibroblast</th>\n",
       "      <th>LifespanUberClock_Tail</th>\n",
       "      <th>LifespanUberClock_panTissue</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ET0215_AF10B</th>\n",
       "      <td>Danio rerio</td>\n",
       "      <td>AF10B</td>\n",
       "      <td>C10</td>\n",
       "      <td>32002151</td>\n",
       "      <td>ET0215_AF10B</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.069660e+11</td>\n",
       "      <td>R03C01</td>\n",
       "      <td>female</td>\n",
       "      <td>Musmusculus</td>\n",
       "      <td>...</td>\n",
       "      <td>23.884253</td>\n",
       "      <td>-6.473401</td>\n",
       "      <td>1.240207</td>\n",
       "      <td>3.801444</td>\n",
       "      <td>0.742179</td>\n",
       "      <td>-3.689742</td>\n",
       "      <td>4.301981</td>\n",
       "      <td>21.046977</td>\n",
       "      <td>5.807267</td>\n",
       "      <td>-7.848188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ET0215_AF11B</th>\n",
       "      <td>Rattus norvegicus</td>\n",
       "      <td>AF11B</td>\n",
       "      <td>B10</td>\n",
       "      <td>32002151</td>\n",
       "      <td>ET0215_AF11B</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.069660e+11</td>\n",
       "      <td>R02C01</td>\n",
       "      <td>female</td>\n",
       "      <td>Musmusculus</td>\n",
       "      <td>...</td>\n",
       "      <td>23.577918</td>\n",
       "      <td>-6.077609</td>\n",
       "      <td>2.016486</td>\n",
       "      <td>4.237305</td>\n",
       "      <td>1.433653</td>\n",
       "      <td>-3.129090</td>\n",
       "      <td>4.888675</td>\n",
       "      <td>21.019877</td>\n",
       "      <td>5.713282</td>\n",
       "      <td>-7.372311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ET0215_AF12L</th>\n",
       "      <td>Mus musculus</td>\n",
       "      <td>AF12L</td>\n",
       "      <td>D05</td>\n",
       "      <td>32002151</td>\n",
       "      <td>ET0215_AF12L</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.069660e+11</td>\n",
       "      <td>R06C02</td>\n",
       "      <td>female</td>\n",
       "      <td>Musmusculus</td>\n",
       "      <td>...</td>\n",
       "      <td>1.350837</td>\n",
       "      <td>2.473628</td>\n",
       "      <td>-0.061754</td>\n",
       "      <td>0.087752</td>\n",
       "      <td>0.792230</td>\n",
       "      <td>0.904557</td>\n",
       "      <td>0.235447</td>\n",
       "      <td>-1.173817</td>\n",
       "      <td>0.387865</td>\n",
       "      <td>0.056342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ET0215_AF1L</th>\n",
       "      <td>Mus musculus</td>\n",
       "      <td>AF1L</td>\n",
       "      <td>H04</td>\n",
       "      <td>32002151</td>\n",
       "      <td>ET0215_AF1L</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.069660e+11</td>\n",
       "      <td>R02C02</td>\n",
       "      <td>female</td>\n",
       "      <td>Musmusculus</td>\n",
       "      <td>...</td>\n",
       "      <td>1.623943</td>\n",
       "      <td>2.530734</td>\n",
       "      <td>0.214300</td>\n",
       "      <td>-0.026863</td>\n",
       "      <td>0.720163</td>\n",
       "      <td>1.010883</td>\n",
       "      <td>0.183221</td>\n",
       "      <td>-0.669693</td>\n",
       "      <td>0.415750</td>\n",
       "      <td>-0.279443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ET0215_AF3B</th>\n",
       "      <td>Mus musculus</td>\n",
       "      <td>AF3B</td>\n",
       "      <td>D06</td>\n",
       "      <td>32002152</td>\n",
       "      <td>ET0215_AF3B</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.069660e+11</td>\n",
       "      <td>R02C02</td>\n",
       "      <td>female</td>\n",
       "      <td>Musmusculus</td>\n",
       "      <td>...</td>\n",
       "      <td>2.239526</td>\n",
       "      <td>1.727559</td>\n",
       "      <td>0.858695</td>\n",
       "      <td>0.550976</td>\n",
       "      <td>1.563069</td>\n",
       "      <td>2.324138</td>\n",
       "      <td>0.213193</td>\n",
       "      <td>0.408538</td>\n",
       "      <td>0.208866</td>\n",
       "      <td>-0.284926</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 91 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             PredictedSpeciesLatinName ExternalSampleID Well_Address  \\\n",
       "SID                                                                    \n",
       "ET0215_AF10B               Danio rerio            AF10B          C10   \n",
       "ET0215_AF11B         Rattus norvegicus            AF11B          B10   \n",
       "ET0215_AF12L              Mus musculus            AF12L          D05   \n",
       "ET0215_AF1L               Mus musculus             AF1L          H04   \n",
       "ET0215_AF3B               Mus musculus             AF3B          D06   \n",
       "\n",
       "              Plate_Number  Sample_Group  Pool_ID    Sentrix_ID  \\\n",
       "SID                                                               \n",
       "ET0215_AF10B      32002151  ET0215_AF10B      NaN  2.069660e+11   \n",
       "ET0215_AF11B      32002151  ET0215_AF11B      NaN  2.069660e+11   \n",
       "ET0215_AF12L      32002151  ET0215_AF12L      NaN  2.069660e+11   \n",
       "ET0215_AF1L       32002151   ET0215_AF1L      NaN  2.069660e+11   \n",
       "ET0215_AF3B       32002152   ET0215_AF3B      NaN  2.069660e+11   \n",
       "\n",
       "             Sentrix_Position     Sex      Species  ...  \\\n",
       "SID                                                 ...   \n",
       "ET0215_AF10B           R03C01  female  Musmusculus  ...   \n",
       "ET0215_AF11B           R02C01  female  Musmusculus  ...   \n",
       "ET0215_AF12L           R06C02  female  Musmusculus  ...   \n",
       "ET0215_AF1L            R02C02  female  Musmusculus  ...   \n",
       "ET0215_AF3B            R02C02  female  Musmusculus  ...   \n",
       "\n",
       "              LifespanUberClock_Heart LifespanUberClock_Kidney  \\\n",
       "SID                                                              \n",
       "ET0215_AF10B                23.884253                -6.473401   \n",
       "ET0215_AF11B                23.577918                -6.077609   \n",
       "ET0215_AF12L                 1.350837                 2.473628   \n",
       "ET0215_AF1L                  1.623943                 2.530734   \n",
       "ET0215_AF3B                  2.239526                 1.727559   \n",
       "\n",
       "             LifespanUberClock_Muscle LifespanUberClock_Cortex  \\\n",
       "SID                                                              \n",
       "ET0215_AF10B                 1.240207                 3.801444   \n",
       "ET0215_AF11B                 2.016486                 4.237305   \n",
       "ET0215_AF12L                -0.061754                 0.087752   \n",
       "ET0215_AF1L                  0.214300                -0.026863   \n",
       "ET0215_AF3B                  0.858695                 0.550976   \n",
       "\n",
       "              LifespanUberClock_Striatum LifespanUberClock_Cerebellum  \\\n",
       "SID                                                                     \n",
       "ET0215_AF10B                    0.742179                    -3.689742   \n",
       "ET0215_AF11B                    1.433653                    -3.129090   \n",
       "ET0215_AF12L                    0.792230                     0.904557   \n",
       "ET0215_AF1L                     0.720163                     1.010883   \n",
       "ET0215_AF3B                     1.563069                     2.324138   \n",
       "\n",
       "              LifespanUberClock_Brain  LifespanUberClock_Fibroblast  \\\n",
       "SID                                                                   \n",
       "ET0215_AF10B                 4.301981                     21.046977   \n",
       "ET0215_AF11B                 4.888675                     21.019877   \n",
       "ET0215_AF12L                 0.235447                     -1.173817   \n",
       "ET0215_AF1L                  0.183221                     -0.669693   \n",
       "ET0215_AF3B                  0.213193                      0.408538   \n",
       "\n",
       "              LifespanUberClock_Tail  LifespanUberClock_panTissue  \n",
       "SID                                                                \n",
       "ET0215_AF10B                5.807267                    -7.848188  \n",
       "ET0215_AF11B                5.713282                    -7.372311  \n",
       "ET0215_AF12L                0.387865                     0.056342  \n",
       "ET0215_AF1L                 0.415750                    -0.279443  \n",
       "ET0215_AF3B                 0.208866                    -0.284926  \n",
       "\n",
       "[5 rows x 91 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_meta = pd.read_csv(gl_metainfo_path, index_col=0)\n",
    "# df_meta.set_index('Basename', inplace=True)\n",
    "# print(df_meta.Tissue.value_counts())\n",
    "print(df_meta.shape)\n",
    "df_meta.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8321695b-ebb2-437c-86de-2ffa97e64c5a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "6c6f56e4-d48d-453a-80af-ba9e6e881f8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# np.intersect1d(df_meta.SID.values, df_p_sesame.columns).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "7e16e2e6-a649-48ea-80c6-0dcd3ca3a3aa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(288, 93)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'SID' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'PredictedSpeciesLatinName' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'ExternalSampleID' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Well_Address' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Sample_Group' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Sentrix_Position' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Sex' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Species' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Tissue' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Strain' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Treatment' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Basename' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'predictedSexByCF' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'PredictedTissue' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Sex.PanMammalianPredictor' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Data Type' as categorical\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def try_replica(x):\n",
    "    try:\n",
    "        return int(x.split('_')[-1])\n",
    "    except:\n",
    "        return 1\n",
    "\n",
    "df_meta_2 = df_meta.copy()\n",
    "df_meta.reset_index(inplace=True)\n",
    "df_meta_2.reset_index(inplace=True)\n",
    "df_meta['Name PerCov'] = df_meta['SID'].str.cat(['percentage'] * df_meta.shape[0], sep='_')\n",
    "df_meta['Data Type'] = 'percentage'\n",
    "df_meta_2['Name PerCov'] = df_meta_2['SID'].str.cat(['coverage'] * df_meta.shape[0], sep='_')\n",
    "df_meta_2['Data Type'] = 'coverage'\n",
    "df_meta['Name PerCov'] = df_meta['SID'].str.cat(['percentage'] * df_meta.shape[0], sep='_')\n",
    "df_meta = pd.concat((df_meta, df_meta_2), axis=0)\n",
    "df_meta.set_index('Name PerCov', inplace=True)\n",
    "print(df_meta.shape)\n",
    "df_meta.head()\n",
    "\n",
    "df_sesame.columns = [x + '_percentage' for x in df_sesame.columns]\n",
    "df_p_sesame.columns = [x + '_coverage' for x in df_p_sesame.columns]\n",
    "\n",
    "df_p_sesame = 10.*(df_p_sesame < 0.01)\n",
    "\n",
    "df_full = pd.concat((df_sesame, df_p_sesame), axis=1)\n",
    "columns = np.sort(df_full.columns)[::-1]\n",
    "df_full = df_full.loc[:,columns]\n",
    "\n",
    "ad = sc.AnnData(df_full)\n",
    "ad.var = pd.concat((ad.var, df_meta.loc[ad.var.index,:]), axis=1)\n",
    "ad.X = sp.csr_matrix(np.nan_to_num(ad.X.copy()))\n",
    "ad.write_h5ad(Path.joinpath(project_dir, \"{}_sesame_with_pvals_wmeta.h5ad\".format(project_id)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "983811ae-8119-4182-b2e7-6c5422fdb055",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "27cbe971-8edb-41a7-a00d-b2de7717f54c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SID</th>\n",
       "      <th>PredictedSpeciesLatinName</th>\n",
       "      <th>ExternalSampleID</th>\n",
       "      <th>Well_Address</th>\n",
       "      <th>Plate_Number</th>\n",
       "      <th>Sample_Group</th>\n",
       "      <th>Pool_ID</th>\n",
       "      <th>Sentrix_ID</th>\n",
       "      <th>Sentrix_Position</th>\n",
       "      <th>Sex</th>\n",
       "      <th>...</th>\n",
       "      <th>LifespanUberClock_Heart</th>\n",
       "      <th>LifespanUberClock_Kidney</th>\n",
       "      <th>LifespanUberClock_Muscle</th>\n",
       "      <th>LifespanUberClock_Cortex</th>\n",
       "      <th>LifespanUberClock_Striatum</th>\n",
       "      <th>LifespanUberClock_Cerebellum</th>\n",
       "      <th>LifespanUberClock_Brain</th>\n",
       "      <th>LifespanUberClock_Fibroblast</th>\n",
       "      <th>LifespanUberClock_Tail</th>\n",
       "      <th>LifespanUberClock_panTissue</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ET0215_AF10B</td>\n",
       "      <td>Danio rerio</td>\n",
       "      <td>AF10B</td>\n",
       "      <td>C10</td>\n",
       "      <td>32002151</td>\n",
       "      <td>ET0215_AF10B</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.069660e+11</td>\n",
       "      <td>R03C01</td>\n",
       "      <td>female</td>\n",
       "      <td>...</td>\n",
       "      <td>23.884253</td>\n",
       "      <td>-6.473401</td>\n",
       "      <td>1.240207</td>\n",
       "      <td>3.801444</td>\n",
       "      <td>0.742179</td>\n",
       "      <td>-3.689742</td>\n",
       "      <td>4.301981</td>\n",
       "      <td>21.046977</td>\n",
       "      <td>5.807267</td>\n",
       "      <td>-7.848188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ET0215_AF11B</td>\n",
       "      <td>Rattus norvegicus</td>\n",
       "      <td>AF11B</td>\n",
       "      <td>B10</td>\n",
       "      <td>32002151</td>\n",
       "      <td>ET0215_AF11B</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.069660e+11</td>\n",
       "      <td>R02C01</td>\n",
       "      <td>female</td>\n",
       "      <td>...</td>\n",
       "      <td>23.577918</td>\n",
       "      <td>-6.077609</td>\n",
       "      <td>2.016486</td>\n",
       "      <td>4.237305</td>\n",
       "      <td>1.433653</td>\n",
       "      <td>-3.129090</td>\n",
       "      <td>4.888675</td>\n",
       "      <td>21.019877</td>\n",
       "      <td>5.713282</td>\n",
       "      <td>-7.372311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ET0215_AF12L</td>\n",
       "      <td>Mus musculus</td>\n",
       "      <td>AF12L</td>\n",
       "      <td>D05</td>\n",
       "      <td>32002151</td>\n",
       "      <td>ET0215_AF12L</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.069660e+11</td>\n",
       "      <td>R06C02</td>\n",
       "      <td>female</td>\n",
       "      <td>...</td>\n",
       "      <td>1.350837</td>\n",
       "      <td>2.473628</td>\n",
       "      <td>-0.061754</td>\n",
       "      <td>0.087752</td>\n",
       "      <td>0.792230</td>\n",
       "      <td>0.904557</td>\n",
       "      <td>0.235447</td>\n",
       "      <td>-1.173817</td>\n",
       "      <td>0.387865</td>\n",
       "      <td>0.056342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ET0215_AF1L</td>\n",
       "      <td>Mus musculus</td>\n",
       "      <td>AF1L</td>\n",
       "      <td>H04</td>\n",
       "      <td>32002151</td>\n",
       "      <td>ET0215_AF1L</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.069660e+11</td>\n",
       "      <td>R02C02</td>\n",
       "      <td>female</td>\n",
       "      <td>...</td>\n",
       "      <td>1.623943</td>\n",
       "      <td>2.530734</td>\n",
       "      <td>0.214300</td>\n",
       "      <td>-0.026863</td>\n",
       "      <td>0.720163</td>\n",
       "      <td>1.010883</td>\n",
       "      <td>0.183221</td>\n",
       "      <td>-0.669693</td>\n",
       "      <td>0.415750</td>\n",
       "      <td>-0.279443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ET0215_AF3B</td>\n",
       "      <td>Mus musculus</td>\n",
       "      <td>AF3B</td>\n",
       "      <td>D06</td>\n",
       "      <td>32002152</td>\n",
       "      <td>ET0215_AF3B</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.069660e+11</td>\n",
       "      <td>R02C02</td>\n",
       "      <td>female</td>\n",
       "      <td>...</td>\n",
       "      <td>2.239526</td>\n",
       "      <td>1.727559</td>\n",
       "      <td>0.858695</td>\n",
       "      <td>0.550976</td>\n",
       "      <td>1.563069</td>\n",
       "      <td>2.324138</td>\n",
       "      <td>0.213193</td>\n",
       "      <td>0.408538</td>\n",
       "      <td>0.208866</td>\n",
       "      <td>-0.284926</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 92 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            SID PredictedSpeciesLatinName ExternalSampleID Well_Address  \\\n",
       "0  ET0215_AF10B               Danio rerio            AF10B          C10   \n",
       "1  ET0215_AF11B         Rattus norvegicus            AF11B          B10   \n",
       "2  ET0215_AF12L              Mus musculus            AF12L          D05   \n",
       "3   ET0215_AF1L              Mus musculus             AF1L          H04   \n",
       "4   ET0215_AF3B              Mus musculus             AF3B          D06   \n",
       "\n",
       "   Plate_Number  Sample_Group  Pool_ID    Sentrix_ID Sentrix_Position     Sex  \\\n",
       "0      32002151  ET0215_AF10B      NaN  2.069660e+11           R03C01  female   \n",
       "1      32002151  ET0215_AF11B      NaN  2.069660e+11           R02C01  female   \n",
       "2      32002151  ET0215_AF12L      NaN  2.069660e+11           R06C02  female   \n",
       "3      32002151   ET0215_AF1L      NaN  2.069660e+11           R02C02  female   \n",
       "4      32002152   ET0215_AF3B      NaN  2.069660e+11           R02C02  female   \n",
       "\n",
       "   ... LifespanUberClock_Heart  LifespanUberClock_Kidney  \\\n",
       "0  ...               23.884253                 -6.473401   \n",
       "1  ...               23.577918                 -6.077609   \n",
       "2  ...                1.350837                  2.473628   \n",
       "3  ...                1.623943                  2.530734   \n",
       "4  ...                2.239526                  1.727559   \n",
       "\n",
       "  LifespanUberClock_Muscle LifespanUberClock_Cortex  \\\n",
       "0                 1.240207                 3.801444   \n",
       "1                 2.016486                 4.237305   \n",
       "2                -0.061754                 0.087752   \n",
       "3                 0.214300                -0.026863   \n",
       "4                 0.858695                 0.550976   \n",
       "\n",
       "  LifespanUberClock_Striatum  LifespanUberClock_Cerebellum  \\\n",
       "0                   0.742179                     -3.689742   \n",
       "1                   1.433653                     -3.129090   \n",
       "2                   0.792230                      0.904557   \n",
       "3                   0.720163                      1.010883   \n",
       "4                   1.563069                      2.324138   \n",
       "\n",
       "  LifespanUberClock_Brain  LifespanUberClock_Fibroblast  \\\n",
       "0                4.301981                     21.046977   \n",
       "1                4.888675                     21.019877   \n",
       "2                0.235447                     -1.173817   \n",
       "3                0.183221                     -0.669693   \n",
       "4                0.213193                      0.408538   \n",
       "\n",
       "   LifespanUberClock_Tail  LifespanUberClock_panTissue  \n",
       "0                5.807267                    -7.848188  \n",
       "1                5.713282                    -7.372311  \n",
       "2                0.387865                     0.056342  \n",
       "3                0.415750                    -0.279443  \n",
       "4                0.208866                    -0.284926  \n",
       "\n",
       "[5 rows x 92 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_anno = pd.read_csv(gl_metainfo_path)\n",
    "# df_anno.set_index('CGid', inplace=True)\n",
    "# df_anno.dropna(axis=0, inplace=True, how='all')\n",
    "# print(df_anno.shape)\n",
    "df_anno.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "406286b9-f9c6-4f10-ac8d-9af06975af81",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OrderedDict([(None,                      cg00101675_BC21  cg00116289_BC21  cg00211372_TC21  \\\n",
      "206966350009_R01C01         0.807241         0.887712         0.930338   \n",
      "206966350009_R02C01         0.806692         0.897543         0.930270   \n",
      "206966350009_R03C01         0.818084         0.871886         0.785316   \n",
      "206966350009_R04C01         0.838988         0.893215         0.888206   \n",
      "206966350009_R05C01         0.859570         0.918056         0.857248   \n",
      "...                              ...              ...              ...   \n",
      "206966350081_R02C02         0.924380         0.907083         0.911996   \n",
      "206966350081_R03C02         0.722618         0.792123         0.884213   \n",
      "206966350081_R04C02         0.923939         0.861986         0.911948   \n",
      "206966350081_R05C02         0.876501         0.892726         0.905180   \n",
      "206966350081_R06C02         0.922529         0.911581         0.874273   \n",
      "\n",
      "                     cg00531009_BC21  cg00747726_TC21  cg00896209_TC21  \\\n",
      "206966350009_R01C01         0.864641         0.747866         0.867096   \n",
      "206966350009_R02C01         0.853007         0.725591         0.866494   \n",
      "206966350009_R03C01         0.867609         0.816723         0.874336   \n",
      "206966350009_R04C01         0.885660         0.695958         0.862897   \n",
      "206966350009_R05C01         0.871312         0.737316         0.843572   \n",
      "...                              ...              ...              ...   \n",
      "206966350081_R02C02         0.868571         0.737919         0.876278   \n",
      "206966350081_R03C02         0.803004         0.617061         0.894057   \n",
      "206966350081_R04C02         0.816946         0.769525         0.898666   \n",
      "206966350081_R05C02         0.841741         0.781550         0.908950   \n",
      "206966350081_R06C02         0.850490         0.793501         0.885702   \n",
      "\n",
      "                     cg00910419_BC21  cg00920372_BC21  cg01326836_BC21  \\\n",
      "206966350009_R01C01         0.032138         0.646722         0.118234   \n",
      "206966350009_R02C01         0.035325         0.671354         0.122473   \n",
      "206966350009_R03C01         0.044873         0.553792         0.136215   \n",
      "206966350009_R04C01         0.039361         0.747593         0.130551   \n",
      "206966350009_R05C01         0.037723         0.680150         0.118172   \n",
      "...                              ...              ...              ...   \n",
      "206966350081_R02C02         0.048091         0.391890         0.156031   \n",
      "206966350081_R03C02         0.044740         0.367051         0.134371   \n",
      "206966350081_R04C02         0.042028         0.472124         0.146655   \n",
      "206966350081_R05C02         0.044384         0.440554         0.151641   \n",
      "206966350081_R06C02         0.037965         0.485603         0.129677   \n",
      "\n",
      "                     cg01340195_BC21  ...  cg22715479_TOPC21  \\\n",
      "206966350009_R01C01         0.823891  ...           0.707431   \n",
      "206966350009_R02C01         0.808850  ...           0.711082   \n",
      "206966350009_R03C01         0.827014  ...           0.701909   \n",
      "206966350009_R04C01         0.834580  ...           0.683461   \n",
      "206966350009_R05C01         0.830774  ...           0.714353   \n",
      "...                              ...  ...                ...   \n",
      "206966350081_R02C02         0.889997  ...           0.679150   \n",
      "206966350081_R03C02         0.727771  ...           0.582417   \n",
      "206966350081_R04C02         0.876258  ...           0.678046   \n",
      "206966350081_R05C02         0.894734  ...           0.693285   \n",
      "206966350081_R06C02         0.914053  ...           0.722444   \n",
      "\n",
      "                     cg12885130_TOPC21  cg17092784_TOPC21  cg18384387_TOPC26  \\\n",
      "206966350009_R01C01           0.892825           0.909506           0.902579   \n",
      "206966350009_R02C01           0.887213           0.878541           0.928872   \n",
      "206966350009_R03C01           0.850067           0.831826           0.903303   \n",
      "206966350009_R04C01           0.917615           0.871034           0.919735   \n",
      "206966350009_R05C01           0.839274           0.883235           0.925295   \n",
      "...                                ...                ...                ...   \n",
      "206966350081_R02C02           0.703187           0.869073           0.900328   \n",
      "206966350081_R03C02           0.541924           0.840895           0.835088   \n",
      "206966350081_R04C02           0.753428           0.831845           0.901353   \n",
      "206966350081_R05C02           0.689797           0.846803           0.864027   \n",
      "206966350081_R06C02           0.656232           0.864620           0.916465   \n",
      "\n",
      "                     cg14306227_TOPC21  cg04603718_TOPC21  cg14891941_BC21  \\\n",
      "206966350009_R01C01           0.799117           0.619297         0.634873   \n",
      "206966350009_R02C01           0.731617           0.627481         0.678719   \n",
      "206966350009_R03C01           0.797510           0.616955         0.645913   \n",
      "206966350009_R04C01           0.803835           0.657069         0.687778   \n",
      "206966350009_R05C01           0.769456           0.619434         0.727961   \n",
      "...                                ...                ...              ...   \n",
      "206966350081_R02C02           0.933067           0.683027         0.590578   \n",
      "206966350081_R03C02           0.841534           0.545075         0.555438   \n",
      "206966350081_R04C02           0.930974           0.686941         0.613574   \n",
      "206966350081_R05C02           0.922936           0.678763         0.568760   \n",
      "206966350081_R06C02           0.923615           0.682126         0.565951   \n",
      "\n",
      "                     cg10950442_TC21  cg17649419_BC21  cg16224928_BC21  \n",
      "206966350009_R01C01         0.940857         0.922442         0.511859  \n",
      "206966350009_R02C01         0.925158         0.954968         0.497846  \n",
      "206966350009_R03C01         0.917054         0.889622         0.476029  \n",
      "206966350009_R04C01         0.929778         0.899515         0.471767  \n",
      "206966350009_R05C01         0.923983         0.932786         0.471488  \n",
      "...                              ...              ...              ...  \n",
      "206966350081_R02C02         0.952380         0.952177         0.487105  \n",
      "206966350081_R03C02         0.910439         0.909849         0.517037  \n",
      "206966350081_R04C02         0.938789         0.955242         0.495805  \n",
      "206966350081_R05C02         0.946587         0.940378         0.462295  \n",
      "206966350081_R06C02         0.949152         0.923470         0.501089  \n",
      "\n",
      "[144 rows x 326732 columns])])\n"
     ]
    },
    {
     "ename": "KeyError",
     "evalue": "'normalized_betas_sesame'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "Input \u001b[0;32mIn [10]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m rdata_sesame \u001b[38;5;241m=\u001b[39m pyreadr\u001b[38;5;241m.\u001b[39mread_r(Path\u001b[38;5;241m.\u001b[39mjoinpath(project_dir, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdetection_p_values_sesame/all_probes_sesame_normalized.RDS\u001b[39m\u001b[38;5;124m'\u001b[39m))\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28mprint\u001b[39m(rdata_sesame)\n\u001b[0;32m----> 3\u001b[0m df_sesame \u001b[38;5;241m=\u001b[39m \u001b[43mrdata_sesame\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mnormalized_betas_sesame\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[1;32m      4\u001b[0m df_sesame\u001b[38;5;241m.\u001b[39mset_index(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mCGid\u001b[39m\u001b[38;5;124m'\u001b[39m, inplace\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m      5\u001b[0m \u001b[38;5;28mprint\u001b[39m(df_sesame\u001b[38;5;241m.\u001b[39mshape)\n",
      "\u001b[0;31mKeyError\u001b[0m: 'normalized_betas_sesame'"
     ]
    }
   ],
   "source": [
    "rdata_sesame = pyreadr.read_r(Path.joinpath(project_dir, 'detection_p_values_sesame/all_probes_sesame_normalized.RDS'))\n",
    "print(rdata_sesame)\n",
    "df_sesame = rdata_sesame['normalized_betas_sesame']\n",
    "df_sesame.set_index('CGid', inplace=True)\n",
    "print(df_sesame.shape)\n",
    "df_sesame.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13a51bfb-dcdc-4fd3-8d4c-d55399932005",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "698ed743-7a16-42dc-802a-4e56bb5d2bbd",
   "metadata": {},
   "source": [
    "# GSE120132 liver Thompson"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "82eeb35e-a03e-437a-882f-cad5af08c42b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "549\n"
     ]
    }
   ],
   "source": [
    "\n",
    "cwd = Path.cwd()\n",
    "project_dir = Path('data/GSE120132')\n",
    "project_id = 'GSE120132'\n",
    "project_samples_dir = 'GSE120132_RAW'\n",
    "path_samples = Path.joinpath(project_dir, project_samples_dir)\n",
    "gl_path_samples = Path.joinpath(cwd, path_samples)\n",
    "gl_project_dir = Path.joinpath(cwd, project_dir)\n",
    "all_samples = os.listdir(gl_path_samples)\n",
    "print(len(all_samples))\n",
    "df_glob = None\n",
    "sample_path_tuples = [Path.joinpath(gl_path_samples, file) for file in all_samples]\n",
    "metainfo_path = 'SraRunTable.txt'\n",
    "gl_metainfo_path = Path.joinpath(project_dir, metainfo_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "880d3c6a-671f-43d4-9d58-beb018da05d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "e90eeb4f-160d-4c9d-b1eb-18d428dc7be0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(739, 29)\n",
      "(549, 29)\n",
      "(549, 30)\n",
      "(1098, 31)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Run</th>\n",
       "      <th>Assay Type</th>\n",
       "      <th>AvgSpotLen</th>\n",
       "      <th>Bases</th>\n",
       "      <th>BioProject</th>\n",
       "      <th>BioSample</th>\n",
       "      <th>Bytes</th>\n",
       "      <th>Center Name</th>\n",
       "      <th>Consent</th>\n",
       "      <th>DATASTORE filetype</th>\n",
       "      <th>...</th>\n",
       "      <th>Sample Name</th>\n",
       "      <th>SRA Study</th>\n",
       "      <th>Age</th>\n",
       "      <th>BIOMATERIAL_PROVIDER</th>\n",
       "      <th>sex</th>\n",
       "      <th>source_name</th>\n",
       "      <th>strain</th>\n",
       "      <th>tissue</th>\n",
       "      <th>Age_months</th>\n",
       "      <th>Data Type</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Name PerCov</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>GSM3394173_percentage</th>\n",
       "      <td>SRR7871491</td>\n",
       "      <td>Bisulfite-Seq</td>\n",
       "      <td>101</td>\n",
       "      <td>3988558276</td>\n",
       "      <td>PRJNA491712</td>\n",
       "      <td>SAMN10085155</td>\n",
       "      <td>2014001941</td>\n",
       "      <td>GEO</td>\n",
       "      <td>public</td>\n",
       "      <td>fastq,sra</td>\n",
       "      <td>...</td>\n",
       "      <td>GSM3394173</td>\n",
       "      <td>SRP162034</td>\n",
       "      <td>20.0</td>\n",
       "      <td>UCLA</td>\n",
       "      <td>female</td>\n",
       "      <td>Adipose</td>\n",
       "      <td>C57BL/6J</td>\n",
       "      <td>Adipose</td>\n",
       "      <td>20mo</td>\n",
       "      <td>percentage</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSM3394174_percentage</th>\n",
       "      <td>SRR7871492</td>\n",
       "      <td>Bisulfite-Seq</td>\n",
       "      <td>101</td>\n",
       "      <td>3929241784</td>\n",
       "      <td>PRJNA491712</td>\n",
       "      <td>SAMN10085154</td>\n",
       "      <td>1980343248</td>\n",
       "      <td>GEO</td>\n",
       "      <td>public</td>\n",
       "      <td>fastq,sra</td>\n",
       "      <td>...</td>\n",
       "      <td>GSM3394174</td>\n",
       "      <td>SRP162034</td>\n",
       "      <td>20.0</td>\n",
       "      <td>UCLA</td>\n",
       "      <td>female</td>\n",
       "      <td>Adipose</td>\n",
       "      <td>C57BL/6J</td>\n",
       "      <td>Adipose</td>\n",
       "      <td>20mo</td>\n",
       "      <td>percentage</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSM3394176_percentage</th>\n",
       "      <td>SRR7871494</td>\n",
       "      <td>Bisulfite-Seq</td>\n",
       "      <td>101</td>\n",
       "      <td>2502848377</td>\n",
       "      <td>PRJNA491712</td>\n",
       "      <td>SAMN10085152</td>\n",
       "      <td>1256787319</td>\n",
       "      <td>GEO</td>\n",
       "      <td>public</td>\n",
       "      <td>fastq,sra</td>\n",
       "      <td>...</td>\n",
       "      <td>GSM3394176</td>\n",
       "      <td>SRP162034</td>\n",
       "      <td>20.0</td>\n",
       "      <td>UCLA</td>\n",
       "      <td>female</td>\n",
       "      <td>Adipose</td>\n",
       "      <td>C57BL/6J</td>\n",
       "      <td>Adipose</td>\n",
       "      <td>20mo</td>\n",
       "      <td>percentage</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSM3394177_percentage</th>\n",
       "      <td>SRR7871495</td>\n",
       "      <td>Bisulfite-Seq</td>\n",
       "      <td>101</td>\n",
       "      <td>4614987142</td>\n",
       "      <td>PRJNA491712</td>\n",
       "      <td>SAMN10085151</td>\n",
       "      <td>2336576497</td>\n",
       "      <td>GEO</td>\n",
       "      <td>public</td>\n",
       "      <td>fastq,sra</td>\n",
       "      <td>...</td>\n",
       "      <td>GSM3394177</td>\n",
       "      <td>SRP162034</td>\n",
       "      <td>20.0</td>\n",
       "      <td>UCLA</td>\n",
       "      <td>male</td>\n",
       "      <td>Adipose</td>\n",
       "      <td>C57BL/6J</td>\n",
       "      <td>Adipose</td>\n",
       "      <td>20mo</td>\n",
       "      <td>percentage</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSM3394181_percentage</th>\n",
       "      <td>SRR7871499</td>\n",
       "      <td>Bisulfite-Seq</td>\n",
       "      <td>101</td>\n",
       "      <td>4000050561</td>\n",
       "      <td>PRJNA491712</td>\n",
       "      <td>SAMN10085185</td>\n",
       "      <td>2048006311</td>\n",
       "      <td>GEO</td>\n",
       "      <td>public</td>\n",
       "      <td>fastq,sra</td>\n",
       "      <td>...</td>\n",
       "      <td>GSM3394181</td>\n",
       "      <td>SRP162034</td>\n",
       "      <td>20.0</td>\n",
       "      <td>UCLA</td>\n",
       "      <td>male</td>\n",
       "      <td>Adipose</td>\n",
       "      <td>C57BL/6J</td>\n",
       "      <td>Adipose</td>\n",
       "      <td>20mo</td>\n",
       "      <td>percentage</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                              Run     Assay Type  AvgSpotLen       Bases  \\\n",
       "Name PerCov                                                                \n",
       "GSM3394173_percentage  SRR7871491  Bisulfite-Seq         101  3988558276   \n",
       "GSM3394174_percentage  SRR7871492  Bisulfite-Seq         101  3929241784   \n",
       "GSM3394176_percentage  SRR7871494  Bisulfite-Seq         101  2502848377   \n",
       "GSM3394177_percentage  SRR7871495  Bisulfite-Seq         101  4614987142   \n",
       "GSM3394181_percentage  SRR7871499  Bisulfite-Seq         101  4000050561   \n",
       "\n",
       "                        BioProject     BioSample       Bytes Center Name  \\\n",
       "Name PerCov                                                                \n",
       "GSM3394173_percentage  PRJNA491712  SAMN10085155  2014001941         GEO   \n",
       "GSM3394174_percentage  PRJNA491712  SAMN10085154  1980343248         GEO   \n",
       "GSM3394176_percentage  PRJNA491712  SAMN10085152  1256787319         GEO   \n",
       "GSM3394177_percentage  PRJNA491712  SAMN10085151  2336576497         GEO   \n",
       "GSM3394181_percentage  PRJNA491712  SAMN10085185  2048006311         GEO   \n",
       "\n",
       "                      Consent DATASTORE filetype  ... Sample Name  SRA Study  \\\n",
       "Name PerCov                                       ...                          \n",
       "GSM3394173_percentage  public          fastq,sra  ...  GSM3394173  SRP162034   \n",
       "GSM3394174_percentage  public          fastq,sra  ...  GSM3394174  SRP162034   \n",
       "GSM3394176_percentage  public          fastq,sra  ...  GSM3394176  SRP162034   \n",
       "GSM3394177_percentage  public          fastq,sra  ...  GSM3394177  SRP162034   \n",
       "GSM3394181_percentage  public          fastq,sra  ...  GSM3394181  SRP162034   \n",
       "\n",
       "                        Age BIOMATERIAL_PROVIDER     sex source_name  \\\n",
       "Name PerCov                                                            \n",
       "GSM3394173_percentage  20.0                 UCLA  female     Adipose   \n",
       "GSM3394174_percentage  20.0                 UCLA  female     Adipose   \n",
       "GSM3394176_percentage  20.0                 UCLA  female     Adipose   \n",
       "GSM3394177_percentage  20.0                 UCLA    male     Adipose   \n",
       "GSM3394181_percentage  20.0                 UCLA    male     Adipose   \n",
       "\n",
       "                         strain   tissue Age_months   Data Type  \n",
       "Name PerCov                                                      \n",
       "GSM3394173_percentage  C57BL/6J  Adipose       20mo  percentage  \n",
       "GSM3394174_percentage  C57BL/6J  Adipose       20mo  percentage  \n",
       "GSM3394176_percentage  C57BL/6J  Adipose       20mo  percentage  \n",
       "GSM3394177_percentage  C57BL/6J  Adipose       20mo  percentage  \n",
       "GSM3394181_percentage  C57BL/6J  Adipose       20mo  percentage  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_meta = pd.read_csv(gl_metainfo_path, sep=',')\n",
    "print(df_meta.shape)\n",
    "df_meta.drop_duplicates('Sample Name', inplace=True)\n",
    "print(df_meta.shape)\n",
    "df_meta['Age_months'] = df_meta['Age'].copy()\n",
    "df_meta['Age_months'].fillna('0mo', inplace=True)\n",
    "df_meta['Age'] = df_meta['Age_months'].apply(lambda x: float(x.replace('mo', '')))\n",
    "print(df_meta.shape)\n",
    "df_meta_2 = df_meta.copy()\n",
    "df_meta['Name PerCov'] = df_meta['Sample Name'].str.cat(['percentage'] * df_meta.shape[0], sep='_')\n",
    "df_meta['Data Type'] = 'percentage'\n",
    "df_meta_2['Name PerCov'] = df_meta_2['Sample Name'].str.cat(['coverage'] * df_meta.shape[0], sep='_')\n",
    "df_meta_2['Data Type'] = 'coverage'\n",
    "df_meta['Name PerCov'] = df_meta['Sample Name'].str.cat(['percentage'] * df_meta.shape[0], sep='_')\n",
    "df_meta = pd.concat((df_meta, df_meta_2), axis=0)\n",
    "df_meta.set_index('Name PerCov', inplace=True)\n",
    "print(df_meta.shape)\n",
    "df_meta.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0e540719-14c0-4a95-8501-c6b928006d69",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/tarkhov/Dropbox/Harvard/Science/Stochastic aging/data/GSE120132/GSE120132_coverage_10_binned_by_1.h5ad\n"
     ]
    }
   ],
   "source": [
    "data_path = Path.joinpath(cwd, project_dir, \"{}_coverage_10_binned_by_1.h5ad\".format(project_id))\n",
    "print(data_path)\n",
    "ad = sc.read_h5ad(data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "3a979d03-5f12-4bcf-83c9-5ae46f0aed3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_meta.loc[ad.var.index,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "7f4eadca-e09f-40e6-9dbf-22ea1151ccbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "ad.var = pd.concat((ad.var, df_meta.loc[ad.var.index,:]), axis=1)\n",
    "ad.X = ad.X.astype(np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b9c03240-96af-4732-90da-b8207c34f79d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Run' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Assay Type' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'BioProject' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'BioSample' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Center Name' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Consent' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'DATASTORE filetype' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'DATASTORE provider' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'DATASTORE region' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Experiment' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'GEO_Accession (exp)' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Instrument' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'LibraryLayout' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'LibrarySelection' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'LibrarySource' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Organism' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Platform' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'ReleaseDate' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Sample Name' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'SRA Study' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'BIOMATERIAL_PROVIDER' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'sex' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'source_name' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'strain' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'tissue' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Age_months' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Data Type' as categorical\n"
     ]
    }
   ],
   "source": [
    "ad.write_h5ad(Path.joinpath(project_dir, \"{}_wmeta.h5ad\".format(project_id)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad7411ba-7264-4d2d-8c7c-b5b610548e6d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "84d33abd-5cff-48d4-ae2c-6fafb713996b",
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "\"Unable to open object (object 'X' doesn't exist)\"",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "Input \u001b[0;32mIn [8]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[43msc\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_hdf\u001b[49m\u001b[43m(\u001b[49m\u001b[43mPath\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjoinpath\u001b[49m\u001b[43m(\u001b[49m\u001b[43mproject_dir\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mLiver_BL6.h5\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mX\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m      2\u001b[0m x\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_io/read.py:140\u001b[0m, in \u001b[0;36mread_hdf\u001b[0;34m(filename, key)\u001b[0m\n\u001b[1;32m    135\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m    136\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mThe file \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfilename\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m stores the following sheets:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00mkeys\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    137\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCall read/read_hdf5 with one of them.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    138\u001b[0m     )\n\u001b[1;32m    139\u001b[0m \u001b[38;5;66;03m# read array\u001b[39;00m\n\u001b[0;32m--> 140\u001b[0m X \u001b[38;5;241m=\u001b[39m \u001b[43mf\u001b[49m\u001b[43m[\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m]\u001b[49m[()]\n\u001b[1;32m    141\u001b[0m \u001b[38;5;66;03m# try to find row and column names\u001b[39;00m\n\u001b[1;32m    142\u001b[0m rows_cols \u001b[38;5;241m=\u001b[39m [{}, {}]\n",
      "File \u001b[0;32mh5py/_objects.pyx:54\u001b[0m, in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32mh5py/_objects.pyx:55\u001b[0m, in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/py310/lib/python3.10/site-packages/h5py/_hl/group.py:305\u001b[0m, in \u001b[0;36mGroup.__getitem__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m    303\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mInvalid HDF5 object reference\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    304\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(name, (\u001b[38;5;28mbytes\u001b[39m, \u001b[38;5;28mstr\u001b[39m)):\n\u001b[0;32m--> 305\u001b[0m     oid \u001b[38;5;241m=\u001b[39m \u001b[43mh5o\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mid\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_e\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlapl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_lapl\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    306\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    307\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAccessing a group is done with bytes or str, \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    308\u001b[0m                     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m not \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(\u001b[38;5;28mtype\u001b[39m(name)))\n",
      "File \u001b[0;32mh5py/_objects.pyx:54\u001b[0m, in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32mh5py/_objects.pyx:55\u001b[0m, in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32mh5py/h5o.pyx:190\u001b[0m, in \u001b[0;36mh5py.h5o.open\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: \"Unable to open object (object 'X' doesn't exist)\""
     ]
    }
   ],
   "source": [
    "# x = sc.read_hdf(Path.joinpath(project_dir, \"Liver_BL6.h5\"), 'X')\n",
    "# x"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79abc7dd-82f5-44ea-8b93-fcb8ed0fd971",
   "metadata": {},
   "source": [
    "# Horvath clock "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "d56b3637-f0b4-4ab6-9fb3-d9a12b64ffba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(348, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CpGmarker</th>\n",
       "      <th>CoefficientTraining</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(Intercept)</td>\n",
       "      <td>0.577683</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cg00059225</td>\n",
       "      <td>0.245959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>cg00088042</td>\n",
       "      <td>0.182716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>cg00252534</td>\n",
       "      <td>0.168223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cg00297950</td>\n",
       "      <td>0.141758</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     CpGmarker  CoefficientTraining\n",
       "0  (Intercept)             0.577683\n",
       "1   cg00059225             0.245959\n",
       "2   cg00088042             0.182716\n",
       "3   cg00252534             0.168223\n",
       "4   cg00297950             0.141758"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cwd = Path.cwd()\n",
    "metainfo_path = 'Horvath_clock.csv'\n",
    "gl_metainfo_path = Path.joinpath(cwd, metainfo_path)\n",
    "\n",
    "df = pd.read_csv(gl_metainfo_path)\n",
    "df[['CpGmarker', 'CoefficientTraining']].to_csv(Path.joinpath(cwd, 'Horvath_blood_clock.csv'), index=None)\n",
    "df.head()\n",
    "\n",
    "cwd = Path.cwd()\n",
    "metainfo_path = 'Horvath_brain_clock.csv'\n",
    "gl_metainfo_path = Path.joinpath(cwd, metainfo_path)\n",
    "\n",
    "paths = ['Horvath_cpgs_brain.txt', 'Horvath_coefs_brain.txt']\n",
    "df = pd.concat([pd.read_csv(Path.joinpath(cwd, metainfo_path)) for metainfo_path in paths], axis=1)\n",
    "print(df.shape)\n",
    "df.columns = ['CpGmarker', 'CoefficientTraining']\n",
    "df.to_csv(gl_metainfo_path, index=None)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e84e41d-035a-4b26-8f90-211452c57d50",
   "metadata": {},
   "source": [
    "# Fibroblasts (dermis and epidermis) E-MTAB-202"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "916de934-5928-4d26-af09-3f185f9c619b",
   "metadata": {},
   "outputs": [],
   "source": [
    "cwd = Path.cwd()\n",
    "# cwd = Path('/Users/tarkhov/Dropbox/Harvard/Science/Church/')\n",
    "project_dir = Path.joinpath(cwd, Path('data/E-MTAB-202/'))\n",
    "project_id = 'E-MTAB-202'\n",
    "project_samples_dir = 'E-MTAB-202_RAW'\n",
    "metainfo_path = 'E-MTAB-202.sdrf.txt'\n",
    "cpginfo_path = 'GPL8490-65.txt'\n",
    "gl_metainfo_path = Path.joinpath(project_dir, metainfo_path)\n",
    "gl_cpginfo_path = Path.joinpath(project_dir, cpginfo_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "24dcaf84-e124-496f-b432-bf77f992ae22",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50, 32)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Material Type</th>\n",
       "      <th>Characteristics[Organism]</th>\n",
       "      <th>Term Source REF</th>\n",
       "      <th>Term Accession Number</th>\n",
       "      <th>Characteristics[OrganismPart]</th>\n",
       "      <th>Term Source REF.1</th>\n",
       "      <th>Term Accession Number.1</th>\n",
       "      <th>Characteristics[Age]</th>\n",
       "      <th>Unit[TimeUnit]</th>\n",
       "      <th>Characteristics[BioSourceType]</th>\n",
       "      <th>...</th>\n",
       "      <th>Derived Array Data Matrix File</th>\n",
       "      <th>Comment [Derived ArrayExpress FTP file]</th>\n",
       "      <th>Derived Array Data Matrix File.1</th>\n",
       "      <th>Comment [Derived ArrayExpress FTP file].1</th>\n",
       "      <th>Factor Value[AGE]</th>\n",
       "      <th>Unit[TimeUnit].1</th>\n",
       "      <th>Factor Value[LAYER_OF_SKIN]</th>\n",
       "      <th>Term Source REF.3</th>\n",
       "      <th>Term Accession Number.3</th>\n",
       "      <th>Age</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hybridization Name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4274594001_A</th>\n",
       "      <td>organism_part</td>\n",
       "      <td>Homo sapiens</td>\n",
       "      <td>EFO</td>\n",
       "      <td>http://purl.org/obo/owl/NCBITaxon#NCBITaxon_9606</td>\n",
       "      <td>epidermis</td>\n",
       "      <td>EFO</td>\n",
       "      <td>EFO_0000954</td>\n",
       "      <td>26</td>\n",
       "      <td>years</td>\n",
       "      <td>fresh_sample</td>\n",
       "      <td>...</td>\n",
       "      <td>all.raw.txt</td>\n",
       "      <td>ftp://ftp.ebi.ac.uk/pub/databases/microarray/d...</td>\n",
       "      <td>all.qnorm.txt</td>\n",
       "      <td>ftp://ftp.ebi.ac.uk/pub/databases/microarray/d...</td>\n",
       "      <td>26</td>\n",
       "      <td>years</td>\n",
       "      <td>epidermis</td>\n",
       "      <td>EFO</td>\n",
       "      <td>EFO_0000954</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4274594001_C</th>\n",
       "      <td>organism_part</td>\n",
       "      <td>Homo sapiens</td>\n",
       "      <td>EFO</td>\n",
       "      <td>http://purl.org/obo/owl/NCBITaxon#NCBITaxon_9606</td>\n",
       "      <td>epidermis</td>\n",
       "      <td>EFO</td>\n",
       "      <td>EFO_0000954</td>\n",
       "      <td>26</td>\n",
       "      <td>years</td>\n",
       "      <td>fresh_sample</td>\n",
       "      <td>...</td>\n",
       "      <td>all.raw.txt</td>\n",
       "      <td>ftp://ftp.ebi.ac.uk/pub/databases/microarray/d...</td>\n",
       "      <td>all.qnorm.txt</td>\n",
       "      <td>ftp://ftp.ebi.ac.uk/pub/databases/microarray/d...</td>\n",
       "      <td>26</td>\n",
       "      <td>years</td>\n",
       "      <td>epidermis</td>\n",
       "      <td>EFO</td>\n",
       "      <td>EFO_0000954</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4274594001_E</th>\n",
       "      <td>organism_part</td>\n",
       "      <td>Homo sapiens</td>\n",
       "      <td>EFO</td>\n",
       "      <td>http://purl.org/obo/owl/NCBITaxon#NCBITaxon_9606</td>\n",
       "      <td>epidermis</td>\n",
       "      <td>EFO</td>\n",
       "      <td>EFO_0000954</td>\n",
       "      <td>34</td>\n",
       "      <td>years</td>\n",
       "      <td>fresh_sample</td>\n",
       "      <td>...</td>\n",
       "      <td>all.raw.txt</td>\n",
       "      <td>ftp://ftp.ebi.ac.uk/pub/databases/microarray/d...</td>\n",
       "      <td>all.qnorm.txt</td>\n",
       "      <td>ftp://ftp.ebi.ac.uk/pub/databases/microarray/d...</td>\n",
       "      <td>34</td>\n",
       "      <td>years</td>\n",
       "      <td>epidermis</td>\n",
       "      <td>EFO</td>\n",
       "      <td>EFO_0000954</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4274594001_G</th>\n",
       "      <td>organism_part</td>\n",
       "      <td>Homo sapiens</td>\n",
       "      <td>EFO</td>\n",
       "      <td>http://purl.org/obo/owl/NCBITaxon#NCBITaxon_9606</td>\n",
       "      <td>epidermis</td>\n",
       "      <td>EFO</td>\n",
       "      <td>EFO_0000954</td>\n",
       "      <td>35</td>\n",
       "      <td>years</td>\n",
       "      <td>fresh_sample</td>\n",
       "      <td>...</td>\n",
       "      <td>all.raw.txt</td>\n",
       "      <td>ftp://ftp.ebi.ac.uk/pub/databases/microarray/d...</td>\n",
       "      <td>all.qnorm.txt</td>\n",
       "      <td>ftp://ftp.ebi.ac.uk/pub/databases/microarray/d...</td>\n",
       "      <td>35</td>\n",
       "      <td>years</td>\n",
       "      <td>epidermis</td>\n",
       "      <td>EFO</td>\n",
       "      <td>EFO_0000954</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4274594001_I</th>\n",
       "      <td>organism_part</td>\n",
       "      <td>Homo sapiens</td>\n",
       "      <td>EFO</td>\n",
       "      <td>http://purl.org/obo/owl/NCBITaxon#NCBITaxon_9606</td>\n",
       "      <td>epidermis</td>\n",
       "      <td>EFO</td>\n",
       "      <td>EFO_0000954</td>\n",
       "      <td>35</td>\n",
       "      <td>years</td>\n",
       "      <td>fresh_sample</td>\n",
       "      <td>...</td>\n",
       "      <td>all.raw.txt</td>\n",
       "      <td>ftp://ftp.ebi.ac.uk/pub/databases/microarray/d...</td>\n",
       "      <td>all.qnorm.txt</td>\n",
       "      <td>ftp://ftp.ebi.ac.uk/pub/databases/microarray/d...</td>\n",
       "      <td>35</td>\n",
       "      <td>years</td>\n",
       "      <td>epidermis</td>\n",
       "      <td>EFO</td>\n",
       "      <td>EFO_0000954</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    Material Type Characteristics[Organism] Term Source REF  \\\n",
       "Hybridization Name                                                            \n",
       "4274594001_A        organism_part              Homo sapiens             EFO   \n",
       "4274594001_C        organism_part              Homo sapiens             EFO   \n",
       "4274594001_E        organism_part              Homo sapiens             EFO   \n",
       "4274594001_G        organism_part              Homo sapiens             EFO   \n",
       "4274594001_I        organism_part              Homo sapiens             EFO   \n",
       "\n",
       "                                               Term Accession Number  \\\n",
       "Hybridization Name                                                     \n",
       "4274594001_A        http://purl.org/obo/owl/NCBITaxon#NCBITaxon_9606   \n",
       "4274594001_C        http://purl.org/obo/owl/NCBITaxon#NCBITaxon_9606   \n",
       "4274594001_E        http://purl.org/obo/owl/NCBITaxon#NCBITaxon_9606   \n",
       "4274594001_G        http://purl.org/obo/owl/NCBITaxon#NCBITaxon_9606   \n",
       "4274594001_I        http://purl.org/obo/owl/NCBITaxon#NCBITaxon_9606   \n",
       "\n",
       "                   Characteristics[OrganismPart] Term Source REF.1  \\\n",
       "Hybridization Name                                                   \n",
       "4274594001_A                           epidermis               EFO   \n",
       "4274594001_C                           epidermis               EFO   \n",
       "4274594001_E                           epidermis               EFO   \n",
       "4274594001_G                           epidermis               EFO   \n",
       "4274594001_I                           epidermis               EFO   \n",
       "\n",
       "                   Term Accession Number.1  Characteristics[Age]  \\\n",
       "Hybridization Name                                                 \n",
       "4274594001_A                   EFO_0000954                    26   \n",
       "4274594001_C                   EFO_0000954                    26   \n",
       "4274594001_E                   EFO_0000954                    34   \n",
       "4274594001_G                   EFO_0000954                    35   \n",
       "4274594001_I                   EFO_0000954                    35   \n",
       "\n",
       "                   Unit[TimeUnit] Characteristics[BioSourceType]  ...  \\\n",
       "Hybridization Name                                                ...   \n",
       "4274594001_A                years                   fresh_sample  ...   \n",
       "4274594001_C                years                   fresh_sample  ...   \n",
       "4274594001_E                years                   fresh_sample  ...   \n",
       "4274594001_G                years                   fresh_sample  ...   \n",
       "4274594001_I                years                   fresh_sample  ...   \n",
       "\n",
       "                   Derived Array Data Matrix File  \\\n",
       "Hybridization Name                                  \n",
       "4274594001_A                          all.raw.txt   \n",
       "4274594001_C                          all.raw.txt   \n",
       "4274594001_E                          all.raw.txt   \n",
       "4274594001_G                          all.raw.txt   \n",
       "4274594001_I                          all.raw.txt   \n",
       "\n",
       "                              Comment [Derived ArrayExpress FTP file]  \\\n",
       "Hybridization Name                                                      \n",
       "4274594001_A        ftp://ftp.ebi.ac.uk/pub/databases/microarray/d...   \n",
       "4274594001_C        ftp://ftp.ebi.ac.uk/pub/databases/microarray/d...   \n",
       "4274594001_E        ftp://ftp.ebi.ac.uk/pub/databases/microarray/d...   \n",
       "4274594001_G        ftp://ftp.ebi.ac.uk/pub/databases/microarray/d...   \n",
       "4274594001_I        ftp://ftp.ebi.ac.uk/pub/databases/microarray/d...   \n",
       "\n",
       "                   Derived Array Data Matrix File.1  \\\n",
       "Hybridization Name                                    \n",
       "4274594001_A                          all.qnorm.txt   \n",
       "4274594001_C                          all.qnorm.txt   \n",
       "4274594001_E                          all.qnorm.txt   \n",
       "4274594001_G                          all.qnorm.txt   \n",
       "4274594001_I                          all.qnorm.txt   \n",
       "\n",
       "                            Comment [Derived ArrayExpress FTP file].1  \\\n",
       "Hybridization Name                                                      \n",
       "4274594001_A        ftp://ftp.ebi.ac.uk/pub/databases/microarray/d...   \n",
       "4274594001_C        ftp://ftp.ebi.ac.uk/pub/databases/microarray/d...   \n",
       "4274594001_E        ftp://ftp.ebi.ac.uk/pub/databases/microarray/d...   \n",
       "4274594001_G        ftp://ftp.ebi.ac.uk/pub/databases/microarray/d...   \n",
       "4274594001_I        ftp://ftp.ebi.ac.uk/pub/databases/microarray/d...   \n",
       "\n",
       "                   Factor Value[AGE] Unit[TimeUnit].1  \\\n",
       "Hybridization Name                                      \n",
       "4274594001_A                      26            years   \n",
       "4274594001_C                      26            years   \n",
       "4274594001_E                      34            years   \n",
       "4274594001_G                      35            years   \n",
       "4274594001_I                      35            years   \n",
       "\n",
       "                   Factor Value[LAYER_OF_SKIN] Term Source REF.3  \\\n",
       "Hybridization Name                                                 \n",
       "4274594001_A                         epidermis               EFO   \n",
       "4274594001_C                         epidermis               EFO   \n",
       "4274594001_E                         epidermis               EFO   \n",
       "4274594001_G                         epidermis               EFO   \n",
       "4274594001_I                         epidermis               EFO   \n",
       "\n",
       "                   Term Accession Number.3 Age  \n",
       "Hybridization Name                              \n",
       "4274594001_A                   EFO_0000954  26  \n",
       "4274594001_C                   EFO_0000954  26  \n",
       "4274594001_E                   EFO_0000954  34  \n",
       "4274594001_G                   EFO_0000954  35  \n",
       "4274594001_I                   EFO_0000954  35  \n",
       "\n",
       "[5 rows x 32 columns]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_meta = pd.read_csv(gl_metainfo_path, sep='\\t', index_col=0)\n",
    "# df_meta['Source Name'] = df_meta['Source Name'].apply(lambda x: x.split(' ')[0])\n",
    "# df_meta['Age'] = df_meta['Characteristics [age.brain]']\n",
    "df_meta.set_index('Hybridization Name', inplace=True)\n",
    "df_meta['Age'] = df_meta['Characteristics[Age]']\n",
    "print(df_meta.shape)\n",
    "df_meta.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "4195fcba-15c6-4948-98da-cb773d72af93",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(27578, 37)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>IlmnStrand</th>\n",
       "      <th>AddressA_ID</th>\n",
       "      <th>AlleleA_ProbeSeq</th>\n",
       "      <th>AddressB_ID</th>\n",
       "      <th>AlleleB_ProbeSeq</th>\n",
       "      <th>GenomeBuild</th>\n",
       "      <th>Chr</th>\n",
       "      <th>MapInfo</th>\n",
       "      <th>Ploidy</th>\n",
       "      <th>...</th>\n",
       "      <th>Distance_to_TSS</th>\n",
       "      <th>CPG_ISLAND</th>\n",
       "      <th>CPG_ISLAND_LOCATIONS</th>\n",
       "      <th>MIR_CPG_ISLAND</th>\n",
       "      <th>RANGE_GB</th>\n",
       "      <th>RANGE_START</th>\n",
       "      <th>RANGE_END</th>\n",
       "      <th>RANGE_STRAND</th>\n",
       "      <th>GB_ACC</th>\n",
       "      <th>ORF</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cg00000292</th>\n",
       "      <td>cg00000292</td>\n",
       "      <td>TOP</td>\n",
       "      <td>990370</td>\n",
       "      <td>AAACATTAATTACCAACCACTCTTCCAAAAAACACTTACCATTAAA...</td>\n",
       "      <td>6660678</td>\n",
       "      <td>AAACATTAATTACCAACCGCTCTTCCAAAAAACACTTACCATTAAA...</td>\n",
       "      <td>36</td>\n",
       "      <td>16</td>\n",
       "      <td>28797601</td>\n",
       "      <td>diploid</td>\n",
       "      <td>...</td>\n",
       "      <td>291.0</td>\n",
       "      <td>True</td>\n",
       "      <td>16:28797486-28797825</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NC_000016.8</td>\n",
       "      <td>28797486.0</td>\n",
       "      <td>28797825.0</td>\n",
       "      <td>+</td>\n",
       "      <td>NM_173201.2</td>\n",
       "      <td>487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00002426</th>\n",
       "      <td>cg00002426</td>\n",
       "      <td>TOP</td>\n",
       "      <td>6580397</td>\n",
       "      <td>AATATAATAACATTACCTTACCCATCTTATAATCAAACCAAACAAA...</td>\n",
       "      <td>6100343</td>\n",
       "      <td>AATATAATAACATTACCTTACCCGTCTTATAATCAAACCAAACGAA...</td>\n",
       "      <td>36</td>\n",
       "      <td>3</td>\n",
       "      <td>57718583</td>\n",
       "      <td>diploid</td>\n",
       "      <td>...</td>\n",
       "      <td>369.0</td>\n",
       "      <td>True</td>\n",
       "      <td>3:57716811-57718675</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NC_000003.10</td>\n",
       "      <td>57716811.0</td>\n",
       "      <td>57718675.0</td>\n",
       "      <td>+</td>\n",
       "      <td>NM_007159.2</td>\n",
       "      <td>7871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00003994</th>\n",
       "      <td>cg00003994</td>\n",
       "      <td>TOP</td>\n",
       "      <td>7150184</td>\n",
       "      <td>AATAATAATAATACCCCCTATAATACTAACTAACAAACATACCCTC...</td>\n",
       "      <td>7150392</td>\n",
       "      <td>AATAATAATAATACCCCCTATAATACTAACTAACAAACATACCCTC...</td>\n",
       "      <td>36</td>\n",
       "      <td>7</td>\n",
       "      <td>15692387</td>\n",
       "      <td>diploid</td>\n",
       "      <td>...</td>\n",
       "      <td>432.0</td>\n",
       "      <td>True</td>\n",
       "      <td>7:15691512-15693551</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NC_000007.12</td>\n",
       "      <td>15691512.0</td>\n",
       "      <td>15693551.0</td>\n",
       "      <td>-</td>\n",
       "      <td>NM_005924.3</td>\n",
       "      <td>4223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00005847</th>\n",
       "      <td>cg00005847</td>\n",
       "      <td>BOT</td>\n",
       "      <td>4850717</td>\n",
       "      <td>TACTATAATACACCCTATATTTAAAACACTAAACTTACCCCATTAA...</td>\n",
       "      <td>1260113</td>\n",
       "      <td>TACTATAATACACCCTATATTTAAAACACTAAACTTACCCCATTAA...</td>\n",
       "      <td>36</td>\n",
       "      <td>2</td>\n",
       "      <td>176737319</td>\n",
       "      <td>diploid</td>\n",
       "      <td>...</td>\n",
       "      <td>268.0</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NM_006898.4</td>\n",
       "      <td>3232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00006414</th>\n",
       "      <td>cg00006414</td>\n",
       "      <td>BOT</td>\n",
       "      <td>6980731</td>\n",
       "      <td>CTCAAAAACCAAACAAAACAAAACCCCAATACTAATCATTAATAAA...</td>\n",
       "      <td>4280093</td>\n",
       "      <td>CTCGAAAACCGAACAAAACAAAACCCCAATACTAATCGTTAATAAA...</td>\n",
       "      <td>36</td>\n",
       "      <td>7</td>\n",
       "      <td>148453770</td>\n",
       "      <td>diploid</td>\n",
       "      <td>...</td>\n",
       "      <td>671.0</td>\n",
       "      <td>True</td>\n",
       "      <td>7:148453584-148455804</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NC_000007.12</td>\n",
       "      <td>148453584.0</td>\n",
       "      <td>148455804.0</td>\n",
       "      <td>+</td>\n",
       "      <td>NM_020781.2</td>\n",
       "      <td>57541</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 37 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Name IlmnStrand  AddressA_ID  \\\n",
       "ID                                               \n",
       "cg00000292  cg00000292        TOP       990370   \n",
       "cg00002426  cg00002426        TOP      6580397   \n",
       "cg00003994  cg00003994        TOP      7150184   \n",
       "cg00005847  cg00005847        BOT      4850717   \n",
       "cg00006414  cg00006414        BOT      6980731   \n",
       "\n",
       "                                             AlleleA_ProbeSeq  AddressB_ID  \\\n",
       "ID                                                                           \n",
       "cg00000292  AAACATTAATTACCAACCACTCTTCCAAAAAACACTTACCATTAAA...      6660678   \n",
       "cg00002426  AATATAATAACATTACCTTACCCATCTTATAATCAAACCAAACAAA...      6100343   \n",
       "cg00003994  AATAATAATAATACCCCCTATAATACTAACTAACAAACATACCCTC...      7150392   \n",
       "cg00005847  TACTATAATACACCCTATATTTAAAACACTAAACTTACCCCATTAA...      1260113   \n",
       "cg00006414  CTCAAAAACCAAACAAAACAAAACCCCAATACTAATCATTAATAAA...      4280093   \n",
       "\n",
       "                                             AlleleB_ProbeSeq  GenomeBuild  \\\n",
       "ID                                                                           \n",
       "cg00000292  AAACATTAATTACCAACCGCTCTTCCAAAAAACACTTACCATTAAA...           36   \n",
       "cg00002426  AATATAATAACATTACCTTACCCGTCTTATAATCAAACCAAACGAA...           36   \n",
       "cg00003994  AATAATAATAATACCCCCTATAATACTAACTAACAAACATACCCTC...           36   \n",
       "cg00005847  TACTATAATACACCCTATATTTAAAACACTAAACTTACCCCATTAA...           36   \n",
       "cg00006414  CTCGAAAACCGAACAAAACAAAACCCCAATACTAATCGTTAATAAA...           36   \n",
       "\n",
       "           Chr    MapInfo   Ploidy  ... Distance_to_TSS CPG_ISLAND  \\\n",
       "ID                                  ...                              \n",
       "cg00000292  16   28797601  diploid  ...           291.0       True   \n",
       "cg00002426   3   57718583  diploid  ...           369.0       True   \n",
       "cg00003994   7   15692387  diploid  ...           432.0       True   \n",
       "cg00005847   2  176737319  diploid  ...           268.0      False   \n",
       "cg00006414   7  148453770  diploid  ...           671.0       True   \n",
       "\n",
       "             CPG_ISLAND_LOCATIONS MIR_CPG_ISLAND      RANGE_GB  RANGE_START  \\\n",
       "ID                                                                            \n",
       "cg00000292   16:28797486-28797825            NaN   NC_000016.8   28797486.0   \n",
       "cg00002426    3:57716811-57718675            NaN  NC_000003.10   57716811.0   \n",
       "cg00003994    7:15691512-15693551            NaN  NC_000007.12   15691512.0   \n",
       "cg00005847                    NaN            NaN           NaN          NaN   \n",
       "cg00006414  7:148453584-148455804            NaN  NC_000007.12  148453584.0   \n",
       "\n",
       "              RANGE_END RANGE_STRAND       GB_ACC    ORF  \n",
       "ID                                                        \n",
       "cg00000292   28797825.0            +  NM_173201.2    487  \n",
       "cg00002426   57718675.0            +  NM_007159.2   7871  \n",
       "cg00003994   15693551.0            -  NM_005924.3   4223  \n",
       "cg00005847          NaN          NaN  NM_006898.4   3232  \n",
       "cg00006414  148455804.0            +  NM_020781.2  57541  \n",
       "\n",
       "[5 rows x 37 columns]"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_cpg_meta = pd.read_csv(gl_cpginfo_path, skiprows=38,index_col=0, sep='\\t')\n",
    "# df_cpg_meta.set_index('AddressA_ID', inplace=True)\n",
    "ref2cpg = dict(zip(df_cpg_meta['AddressA_ID'].values, df_cpg_meta['Name'].values))\n",
    "cpg2ref = dict(zip(df_cpg_meta['Name'].values, df_cpg_meta['AddressA_ID'].values))\n",
    "print(df_cpg_meta.shape)\n",
    "df_cpg_meta.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "1e4d7100-5344-4858-a246-12a87e3ae0c7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/tarkhov/Dropbox/Harvard/Science/Stochastic aging/data/E-MTAB-202/all.raw.txt\n",
      "(27617, 51)\n",
      "(27577, 50)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>4274594001_A</th>\n",
       "      <th>4274594001_C</th>\n",
       "      <th>4274594001_E</th>\n",
       "      <th>4274594001_G</th>\n",
       "      <th>4274594001_I</th>\n",
       "      <th>4274594001_K</th>\n",
       "      <th>4274594001_B</th>\n",
       "      <th>4274594001_D</th>\n",
       "      <th>4274594001_F</th>\n",
       "      <th>4274594001_H</th>\n",
       "      <th>...</th>\n",
       "      <th>4493586143_A</th>\n",
       "      <th>4493586143_C</th>\n",
       "      <th>4493586143_E</th>\n",
       "      <th>4493586143_G</th>\n",
       "      <th>4493586143_I</th>\n",
       "      <th>4493586143_K</th>\n",
       "      <th>4493586143_B</th>\n",
       "      <th>4493586143_D</th>\n",
       "      <th>4493586143_F</th>\n",
       "      <th>4493586143_H</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cg24621042</th>\n",
       "      <td>0.096770</td>\n",
       "      <td>0.123284</td>\n",
       "      <td>0.100730</td>\n",
       "      <td>0.127998</td>\n",
       "      <td>0.105793</td>\n",
       "      <td>0.155339</td>\n",
       "      <td>0.107464</td>\n",
       "      <td>0.133817</td>\n",
       "      <td>0.128372</td>\n",
       "      <td>0.183879</td>\n",
       "      <td>...</td>\n",
       "      <td>0.151920</td>\n",
       "      <td>0.140854</td>\n",
       "      <td>0.178528</td>\n",
       "      <td>0.110637</td>\n",
       "      <td>0.144080</td>\n",
       "      <td>0.199691</td>\n",
       "      <td>0.143487</td>\n",
       "      <td>0.163701</td>\n",
       "      <td>0.184594</td>\n",
       "      <td>0.119285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg20969242</th>\n",
       "      <td>0.036775</td>\n",
       "      <td>0.049523</td>\n",
       "      <td>0.040535</td>\n",
       "      <td>0.073580</td>\n",
       "      <td>0.064829</td>\n",
       "      <td>0.120619</td>\n",
       "      <td>0.031830</td>\n",
       "      <td>0.054697</td>\n",
       "      <td>0.094011</td>\n",
       "      <td>0.052438</td>\n",
       "      <td>...</td>\n",
       "      <td>0.040699</td>\n",
       "      <td>0.035990</td>\n",
       "      <td>0.031408</td>\n",
       "      <td>0.048080</td>\n",
       "      <td>0.024568</td>\n",
       "      <td>0.027761</td>\n",
       "      <td>0.050397</td>\n",
       "      <td>0.022352</td>\n",
       "      <td>0.030006</td>\n",
       "      <td>0.035396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg12832649</th>\n",
       "      <td>0.022293</td>\n",
       "      <td>0.035831</td>\n",
       "      <td>0.030622</td>\n",
       "      <td>0.044723</td>\n",
       "      <td>0.042642</td>\n",
       "      <td>0.032916</td>\n",
       "      <td>0.059618</td>\n",
       "      <td>0.035992</td>\n",
       "      <td>0.046247</td>\n",
       "      <td>0.046167</td>\n",
       "      <td>...</td>\n",
       "      <td>0.055763</td>\n",
       "      <td>0.031421</td>\n",
       "      <td>0.033014</td>\n",
       "      <td>0.031402</td>\n",
       "      <td>0.027380</td>\n",
       "      <td>0.029348</td>\n",
       "      <td>0.030920</td>\n",
       "      <td>0.036465</td>\n",
       "      <td>0.021631</td>\n",
       "      <td>0.024219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg15789095</th>\n",
       "      <td>0.434040</td>\n",
       "      <td>0.452556</td>\n",
       "      <td>0.633179</td>\n",
       "      <td>0.622742</td>\n",
       "      <td>0.506092</td>\n",
       "      <td>0.552206</td>\n",
       "      <td>0.641472</td>\n",
       "      <td>0.659342</td>\n",
       "      <td>0.637848</td>\n",
       "      <td>0.721747</td>\n",
       "      <td>...</td>\n",
       "      <td>0.682052</td>\n",
       "      <td>0.845148</td>\n",
       "      <td>0.543396</td>\n",
       "      <td>0.645945</td>\n",
       "      <td>0.630410</td>\n",
       "      <td>0.751274</td>\n",
       "      <td>0.456248</td>\n",
       "      <td>0.542699</td>\n",
       "      <td>0.607037</td>\n",
       "      <td>0.580707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg25748127</th>\n",
       "      <td>0.021382</td>\n",
       "      <td>0.040859</td>\n",
       "      <td>0.030028</td>\n",
       "      <td>0.041858</td>\n",
       "      <td>0.048588</td>\n",
       "      <td>0.063735</td>\n",
       "      <td>0.029220</td>\n",
       "      <td>0.033656</td>\n",
       "      <td>0.038266</td>\n",
       "      <td>0.039451</td>\n",
       "      <td>...</td>\n",
       "      <td>0.020628</td>\n",
       "      <td>0.023107</td>\n",
       "      <td>0.023614</td>\n",
       "      <td>0.025893</td>\n",
       "      <td>0.022192</td>\n",
       "      <td>0.016923</td>\n",
       "      <td>0.027560</td>\n",
       "      <td>0.027785</td>\n",
       "      <td>0.032014</td>\n",
       "      <td>0.011861</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 50 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            4274594001_A  4274594001_C  4274594001_E  4274594001_G  \\\n",
       "ID                                                                   \n",
       "cg24621042      0.096770      0.123284      0.100730      0.127998   \n",
       "cg20969242      0.036775      0.049523      0.040535      0.073580   \n",
       "cg12832649      0.022293      0.035831      0.030622      0.044723   \n",
       "cg15789095      0.434040      0.452556      0.633179      0.622742   \n",
       "cg25748127      0.021382      0.040859      0.030028      0.041858   \n",
       "\n",
       "            4274594001_I  4274594001_K  4274594001_B  4274594001_D  \\\n",
       "ID                                                                   \n",
       "cg24621042      0.105793      0.155339      0.107464      0.133817   \n",
       "cg20969242      0.064829      0.120619      0.031830      0.054697   \n",
       "cg12832649      0.042642      0.032916      0.059618      0.035992   \n",
       "cg15789095      0.506092      0.552206      0.641472      0.659342   \n",
       "cg25748127      0.048588      0.063735      0.029220      0.033656   \n",
       "\n",
       "            4274594001_F  4274594001_H  ...  4493586143_A  4493586143_C  \\\n",
       "ID                                      ...                               \n",
       "cg24621042      0.128372      0.183879  ...      0.151920      0.140854   \n",
       "cg20969242      0.094011      0.052438  ...      0.040699      0.035990   \n",
       "cg12832649      0.046247      0.046167  ...      0.055763      0.031421   \n",
       "cg15789095      0.637848      0.721747  ...      0.682052      0.845148   \n",
       "cg25748127      0.038266      0.039451  ...      0.020628      0.023107   \n",
       "\n",
       "            4493586143_E  4493586143_G  4493586143_I  4493586143_K  \\\n",
       "ID                                                                   \n",
       "cg24621042      0.178528      0.110637      0.144080      0.199691   \n",
       "cg20969242      0.031408      0.048080      0.024568      0.027761   \n",
       "cg12832649      0.033014      0.031402      0.027380      0.029348   \n",
       "cg15789095      0.543396      0.645945      0.630410      0.751274   \n",
       "cg25748127      0.023614      0.025893      0.022192      0.016923   \n",
       "\n",
       "            4493586143_B  4493586143_D  4493586143_F  4493586143_H  \n",
       "ID                                                                  \n",
       "cg24621042      0.143487      0.163701      0.184594      0.119285  \n",
       "cg20969242      0.050397      0.022352      0.030006      0.035396  \n",
       "cg12832649      0.030920      0.036465      0.021631      0.024219  \n",
       "cg15789095      0.456248      0.542699      0.607037      0.580707  \n",
       "cg25748127      0.027560      0.027785      0.032014      0.011861  \n",
       "\n",
       "[5 rows x 50 columns]"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_path = Path.joinpath(cwd, project_dir, \"all.raw.txt\")\n",
    "print(data_path)\n",
    "df_percent = pd.read_csv(data_path, sep='\\t', decimal=',',  header=None, skiprows=2)#, index_col=0, header=[0,1,2])\n",
    "cols = pd.read_csv(data_path, sep='\\t', header=0, nrows=1).columns#, index_col=0, header=[0,1,2])\n",
    "df_percent.columns = cols\n",
    "df_percent.drop(labels=0, axis=0, inplace=True)\n",
    "print(df_percent.shape)\n",
    "df_percent = df_percent[df_percent['Hybridization REF'].isin(ref2cpg.keys())]\n",
    "df_percent['ID'] = df_percent['Hybridization REF'].apply(lambda x: ref2cpg[x])\n",
    "df_percent.set_index('ID', inplace=True)\n",
    "df_percent.drop('Hybridization REF', axis=1, inplace=True)\n",
    "# df_percent.columns = df_percent.columns.get_level_values(1)\n",
    "print(df_percent.shape)\n",
    "# df_percent = df_percent.T\n",
    "df_percent.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "90cfe15f-62d5-41e3-b84d-e2e933c9f174",
   "metadata": {},
   "outputs": [],
   "source": [
    "ad = sc.AnnData(df_percent)\n",
    "ad.var = pd.concat((ad.var, df_meta.loc[ad.var.index,:]), axis=1)\n",
    "ad.obs = df_cpg_meta.loc[df_percent.index.values.astype(str),:]\n",
    "ad.X = ad.X.astype(np.float16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "a5f03e75-ede4-467d-9a0a-d1cf6a4d255d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'IlmnStrand' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'AlleleA_ProbeSeq' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'AlleleB_ProbeSeq' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Chr' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Ploidy' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Species' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Source' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'SourceStrand' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'SourceSeq' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'TopGenomicSeq' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Next_Base' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Color_Channel' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Gene_Strand' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Gene_ID' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Symbol' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Synonym' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Accession' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'GID' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Annotation' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Product' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'CPG_ISLAND_LOCATIONS' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'MIR_CPG_ISLAND' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'RANGE_GB' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'RANGE_STRAND' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'GB_ACC' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Material Type' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Characteristics[Organism]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Characteristics[OrganismPart]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.1' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number.1' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Unit[TimeUnit]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Characteristics[BioSourceType]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Characteristics[Sex]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.2' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number.2' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Characteristics[BioSourceProvider]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Protocol REF' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Protocol REF.1' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Array Design REF' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Comment[Array serial number]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Protocol REF.2' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Comment [ArrayExpress FTP file]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Protocol REF.3' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Derived Array Data Matrix File' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Comment [Derived ArrayExpress FTP file]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Derived Array Data Matrix File.1' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Comment [Derived ArrayExpress FTP file].1' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Unit[TimeUnit].1' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Factor Value[LAYER_OF_SKIN]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.3' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number.3' as categorical\n"
     ]
    }
   ],
   "source": [
    "ad.write_h5ad(Path.joinpath(project_dir, \"{}.h5ad\".format(project_id)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f3905a3-c694-42b3-bf3f-5d1fcc9727cc",
   "metadata": {},
   "source": [
    "# Brain aging methylation (GSE59685)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "42add339-9ed9-4b5b-a368-cbfddb90fae0",
   "metadata": {},
   "outputs": [],
   "source": [
    "cwd = Path.cwd()\n",
    "# cwd = Path('/Users/tarkhov/Dropbox/Harvard/Science/Church/')\n",
    "project_dir = Path.joinpath(cwd, Path('data/GSE59685/'))\n",
    "project_id = 'GSE59685'\n",
    "project_samples_dir = 'GSE59685_RAW'\n",
    "metainfo_path = 'E-GEOD-59685.sdrf.txt'\n",
    "gl_metainfo_path = Path.joinpath(project_dir, metainfo_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "fc5d3ddc-bc0c-4fb2-b5d3-92cae6fd3ecd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(531, 49)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Comment [Sample_description]</th>\n",
       "      <th>Comment [Sample_source_name]</th>\n",
       "      <th>Comment [Sample_title]</th>\n",
       "      <th>Characteristics [Sex]</th>\n",
       "      <th>Term Source REF</th>\n",
       "      <th>Term Accession Number</th>\n",
       "      <th>Characteristics [ad.disease.status]</th>\n",
       "      <th>Term Source REF.1</th>\n",
       "      <th>Term Accession Number.1</th>\n",
       "      <th>Characteristics [age.blood]</th>\n",
       "      <th>...</th>\n",
       "      <th>FactorValue [BARCODE]</th>\n",
       "      <th>FactorValue [BRAAK.STAGE]</th>\n",
       "      <th>FactorValue [SEX]</th>\n",
       "      <th>Term Source REF.9</th>\n",
       "      <th>Term Accession Number.5</th>\n",
       "      <th>FactorValue [SOURCE TISSUE]</th>\n",
       "      <th>Term Source REF.10</th>\n",
       "      <th>Term Accession Number.6</th>\n",
       "      <th>FactorValue [SUBJECTID]</th>\n",
       "      <th>Age</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Source Name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>GSM1443800</th>\n",
       "      <td>[[]]</td>\n",
       "      <td>whole blood</td>\n",
       "      <td>whole blood_subject_205</td>\n",
       "      <td>male</td>\n",
       "      <td>EFO</td>\n",
       "      <td>EFO_0001266</td>\n",
       "      <td>C</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>88.0</td>\n",
       "      <td>...</td>\n",
       "      <td>7796806142_R06C01</td>\n",
       "      <td>Exclude</td>\n",
       "      <td>male</td>\n",
       "      <td>EFO</td>\n",
       "      <td>EFO_0001266</td>\n",
       "      <td>whole blood</td>\n",
       "      <td>EFO</td>\n",
       "      <td>http://purl.obolibrary.org/obo/UBERON_0000178</td>\n",
       "      <td>205</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSM1443799</th>\n",
       "      <td>[[]]</td>\n",
       "      <td>whole blood</td>\n",
       "      <td>whole blood_subject_204</td>\n",
       "      <td>female</td>\n",
       "      <td>EFO</td>\n",
       "      <td>EFO_0001265</td>\n",
       "      <td>C</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>72.0</td>\n",
       "      <td>...</td>\n",
       "      <td>7512560157_R04C01</td>\n",
       "      <td>Exclude</td>\n",
       "      <td>female</td>\n",
       "      <td>EFO</td>\n",
       "      <td>EFO_0001265</td>\n",
       "      <td>whole blood</td>\n",
       "      <td>EFO</td>\n",
       "      <td>http://purl.obolibrary.org/obo/UBERON_0000178</td>\n",
       "      <td>204</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSM1443798</th>\n",
       "      <td>[[]]</td>\n",
       "      <td>whole blood</td>\n",
       "      <td>whole blood_subject_203</td>\n",
       "      <td>female</td>\n",
       "      <td>EFO</td>\n",
       "      <td>EFO_0001265</td>\n",
       "      <td>C</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>79.0</td>\n",
       "      <td>...</td>\n",
       "      <td>7512560157_R02C01</td>\n",
       "      <td>Exclude</td>\n",
       "      <td>female</td>\n",
       "      <td>EFO</td>\n",
       "      <td>EFO_0001265</td>\n",
       "      <td>whole blood</td>\n",
       "      <td>EFO</td>\n",
       "      <td>http://purl.obolibrary.org/obo/UBERON_0000178</td>\n",
       "      <td>203</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSM1443797</th>\n",
       "      <td>[[]]</td>\n",
       "      <td>whole blood</td>\n",
       "      <td>whole blood_subject_202</td>\n",
       "      <td>female</td>\n",
       "      <td>EFO</td>\n",
       "      <td>EFO_0001265</td>\n",
       "      <td>C</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>74.0</td>\n",
       "      <td>...</td>\n",
       "      <td>7512551028_R02C02</td>\n",
       "      <td>Exclude</td>\n",
       "      <td>female</td>\n",
       "      <td>EFO</td>\n",
       "      <td>EFO_0001265</td>\n",
       "      <td>whole blood</td>\n",
       "      <td>EFO</td>\n",
       "      <td>http://purl.obolibrary.org/obo/UBERON_0000178</td>\n",
       "      <td>202</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSM1443796</th>\n",
       "      <td>[[]]</td>\n",
       "      <td>whole blood</td>\n",
       "      <td>whole blood_subject_201</td>\n",
       "      <td>male</td>\n",
       "      <td>EFO</td>\n",
       "      <td>EFO_0001266</td>\n",
       "      <td>C</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>76.0</td>\n",
       "      <td>...</td>\n",
       "      <td>7512551028_R05C01</td>\n",
       "      <td>Exclude</td>\n",
       "      <td>male</td>\n",
       "      <td>EFO</td>\n",
       "      <td>EFO_0001266</td>\n",
       "      <td>whole blood</td>\n",
       "      <td>EFO</td>\n",
       "      <td>http://purl.obolibrary.org/obo/UBERON_0000178</td>\n",
       "      <td>201</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 49 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Comment [Sample_description] Comment [Sample_source_name]  \\\n",
       "Source Name                                                             \n",
       "GSM1443800                          [[]]                  whole blood   \n",
       "GSM1443799                          [[]]                  whole blood   \n",
       "GSM1443798                          [[]]                  whole blood   \n",
       "GSM1443797                          [[]]                  whole blood   \n",
       "GSM1443796                          [[]]                  whole blood   \n",
       "\n",
       "              Comment [Sample_title] Characteristics [Sex] Term Source REF  \\\n",
       "Source Name                                                                  \n",
       "GSM1443800   whole blood_subject_205                  male             EFO   \n",
       "GSM1443799   whole blood_subject_204                female             EFO   \n",
       "GSM1443798   whole blood_subject_203                female             EFO   \n",
       "GSM1443797   whole blood_subject_202                female             EFO   \n",
       "GSM1443796   whole blood_subject_201                  male             EFO   \n",
       "\n",
       "            Term Accession Number Characteristics [ad.disease.status]  \\\n",
       "Source Name                                                             \n",
       "GSM1443800            EFO_0001266                                   C   \n",
       "GSM1443799            EFO_0001265                                   C   \n",
       "GSM1443798            EFO_0001265                                   C   \n",
       "GSM1443797            EFO_0001265                                   C   \n",
       "GSM1443796            EFO_0001266                                   C   \n",
       "\n",
       "            Term Source REF.1 Term Accession Number.1  \\\n",
       "Source Name                                             \n",
       "GSM1443800                                              \n",
       "GSM1443799                                              \n",
       "GSM1443798                                              \n",
       "GSM1443797                                              \n",
       "GSM1443796                                              \n",
       "\n",
       "             Characteristics [age.blood]  ...  FactorValue [BARCODE]  \\\n",
       "Source Name                               ...                          \n",
       "GSM1443800                          88.0  ...      7796806142_R06C01   \n",
       "GSM1443799                          72.0  ...      7512560157_R04C01   \n",
       "GSM1443798                          79.0  ...      7512560157_R02C01   \n",
       "GSM1443797                          74.0  ...      7512551028_R02C02   \n",
       "GSM1443796                          76.0  ...      7512551028_R05C01   \n",
       "\n",
       "            FactorValue [BRAAK.STAGE] FactorValue [SEX] Term Source REF.9  \\\n",
       "Source Name                                                                 \n",
       "GSM1443800                    Exclude              male               EFO   \n",
       "GSM1443799                    Exclude            female               EFO   \n",
       "GSM1443798                    Exclude            female               EFO   \n",
       "GSM1443797                    Exclude            female               EFO   \n",
       "GSM1443796                    Exclude              male               EFO   \n",
       "\n",
       "            Term Accession Number.5 FactorValue [SOURCE TISSUE]  \\\n",
       "Source Name                                                       \n",
       "GSM1443800              EFO_0001266                 whole blood   \n",
       "GSM1443799              EFO_0001265                 whole blood   \n",
       "GSM1443798              EFO_0001265                 whole blood   \n",
       "GSM1443797              EFO_0001265                 whole blood   \n",
       "GSM1443796              EFO_0001266                 whole blood   \n",
       "\n",
       "            Term Source REF.10                        Term Accession Number.6  \\\n",
       "Source Name                                                                     \n",
       "GSM1443800                 EFO  http://purl.obolibrary.org/obo/UBERON_0000178   \n",
       "GSM1443799                 EFO  http://purl.obolibrary.org/obo/UBERON_0000178   \n",
       "GSM1443798                 EFO  http://purl.obolibrary.org/obo/UBERON_0000178   \n",
       "GSM1443797                 EFO  http://purl.obolibrary.org/obo/UBERON_0000178   \n",
       "GSM1443796                 EFO  http://purl.obolibrary.org/obo/UBERON_0000178   \n",
       "\n",
       "            FactorValue [SUBJECTID]  Age  \n",
       "Source Name                               \n",
       "GSM1443800                      205  NaN  \n",
       "GSM1443799                      204  NaN  \n",
       "GSM1443798                      203  NaN  \n",
       "GSM1443797                      202  NaN  \n",
       "GSM1443796                      201  NaN  \n",
       "\n",
       "[5 rows x 49 columns]"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_meta = pd.read_csv(gl_metainfo_path, sep='\\t')\n",
    "df_meta['Source Name'] = df_meta['Source Name'].apply(lambda x: x.split(' ')[0])\n",
    "df_meta['Age'] = df_meta['Characteristics [age.brain]']\n",
    "df_meta.set_index('Source Name', inplace=True)\n",
    "print(df_meta.shape)\n",
    "df_meta.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "deefd981-e74c-40bd-abf9-c0f502327c40",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/tarkhov/Dropbox/Harvard/Science/Stochastic aging/data/GSE59685/GSE59685_betas.csv\n",
      "(485577, 531)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GSM1443279</th>\n",
       "      <th>GSM1443663</th>\n",
       "      <th>GSM1443434</th>\n",
       "      <th>GSM1443547</th>\n",
       "      <th>GSM1443577</th>\n",
       "      <th>GSM1443475</th>\n",
       "      <th>GSM1443326</th>\n",
       "      <th>GSM1443640</th>\n",
       "      <th>GSM1443389</th>\n",
       "      <th>GSM1443573</th>\n",
       "      <th>...</th>\n",
       "      <th>GSM1443425</th>\n",
       "      <th>GSM1443711</th>\n",
       "      <th>GSM1443363</th>\n",
       "      <th>GSM1443400</th>\n",
       "      <th>GSM1443470</th>\n",
       "      <th>GSM1443372</th>\n",
       "      <th>GSM1443303</th>\n",
       "      <th>GSM1443617</th>\n",
       "      <th>GSM1443348</th>\n",
       "      <th>GSM1443322</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cg00000029</th>\n",
       "      <td>0.559132</td>\n",
       "      <td>0.620022</td>\n",
       "      <td>0.516109</td>\n",
       "      <td>0.537238</td>\n",
       "      <td>0.552571</td>\n",
       "      <td>0.579338</td>\n",
       "      <td>0.548386</td>\n",
       "      <td>0.567021</td>\n",
       "      <td>0.568738</td>\n",
       "      <td>0.545403</td>\n",
       "      <td>...</td>\n",
       "      <td>0.394504</td>\n",
       "      <td>0.469003</td>\n",
       "      <td>0.408238</td>\n",
       "      <td>0.485401</td>\n",
       "      <td>0.498065</td>\n",
       "      <td>0.554049</td>\n",
       "      <td>0.468078</td>\n",
       "      <td>0.470046</td>\n",
       "      <td>0.312732</td>\n",
       "      <td>0.396636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00000108</th>\n",
       "      <td>0.873277</td>\n",
       "      <td>0.915873</td>\n",
       "      <td>0.917386</td>\n",
       "      <td>0.910278</td>\n",
       "      <td>0.890680</td>\n",
       "      <td>0.905825</td>\n",
       "      <td>0.903829</td>\n",
       "      <td>0.903816</td>\n",
       "      <td>0.867887</td>\n",
       "      <td>0.889484</td>\n",
       "      <td>...</td>\n",
       "      <td>0.905646</td>\n",
       "      <td>0.912011</td>\n",
       "      <td>0.915954</td>\n",
       "      <td>0.871444</td>\n",
       "      <td>0.899956</td>\n",
       "      <td>0.896206</td>\n",
       "      <td>0.903646</td>\n",
       "      <td>0.901356</td>\n",
       "      <td>0.913590</td>\n",
       "      <td>0.895599</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00000109</th>\n",
       "      <td>0.740187</td>\n",
       "      <td>0.756004</td>\n",
       "      <td>0.766046</td>\n",
       "      <td>0.767177</td>\n",
       "      <td>0.725826</td>\n",
       "      <td>0.722050</td>\n",
       "      <td>0.702399</td>\n",
       "      <td>0.760638</td>\n",
       "      <td>0.742862</td>\n",
       "      <td>0.807247</td>\n",
       "      <td>...</td>\n",
       "      <td>0.817154</td>\n",
       "      <td>0.823309</td>\n",
       "      <td>0.842378</td>\n",
       "      <td>0.827500</td>\n",
       "      <td>0.776130</td>\n",
       "      <td>0.829127</td>\n",
       "      <td>0.816067</td>\n",
       "      <td>0.839910</td>\n",
       "      <td>0.802812</td>\n",
       "      <td>0.800367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00000165</th>\n",
       "      <td>0.246977</td>\n",
       "      <td>0.232917</td>\n",
       "      <td>0.238165</td>\n",
       "      <td>0.265406</td>\n",
       "      <td>0.241584</td>\n",
       "      <td>0.247898</td>\n",
       "      <td>0.286702</td>\n",
       "      <td>0.225935</td>\n",
       "      <td>0.269163</td>\n",
       "      <td>0.275686</td>\n",
       "      <td>...</td>\n",
       "      <td>0.241856</td>\n",
       "      <td>0.255883</td>\n",
       "      <td>0.214639</td>\n",
       "      <td>0.249356</td>\n",
       "      <td>0.256304</td>\n",
       "      <td>0.312047</td>\n",
       "      <td>0.263720</td>\n",
       "      <td>0.270058</td>\n",
       "      <td>0.248141</td>\n",
       "      <td>0.273269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00000236</th>\n",
       "      <td>0.815449</td>\n",
       "      <td>0.808496</td>\n",
       "      <td>0.808723</td>\n",
       "      <td>0.797503</td>\n",
       "      <td>0.785904</td>\n",
       "      <td>0.809988</td>\n",
       "      <td>0.796479</td>\n",
       "      <td>0.768113</td>\n",
       "      <td>0.773171</td>\n",
       "      <td>0.808128</td>\n",
       "      <td>...</td>\n",
       "      <td>0.695515</td>\n",
       "      <td>0.707434</td>\n",
       "      <td>0.718788</td>\n",
       "      <td>0.727323</td>\n",
       "      <td>0.702707</td>\n",
       "      <td>0.679252</td>\n",
       "      <td>0.668877</td>\n",
       "      <td>0.708453</td>\n",
       "      <td>0.665294</td>\n",
       "      <td>0.699228</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 531 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            GSM1443279  GSM1443663  GSM1443434  GSM1443547  GSM1443577  \\\n",
       "cg00000029    0.559132    0.620022    0.516109    0.537238    0.552571   \n",
       "cg00000108    0.873277    0.915873    0.917386    0.910278    0.890680   \n",
       "cg00000109    0.740187    0.756004    0.766046    0.767177    0.725826   \n",
       "cg00000165    0.246977    0.232917    0.238165    0.265406    0.241584   \n",
       "cg00000236    0.815449    0.808496    0.808723    0.797503    0.785904   \n",
       "\n",
       "            GSM1443475  GSM1443326  GSM1443640  GSM1443389  GSM1443573  ...  \\\n",
       "cg00000029    0.579338    0.548386    0.567021    0.568738    0.545403  ...   \n",
       "cg00000108    0.905825    0.903829    0.903816    0.867887    0.889484  ...   \n",
       "cg00000109    0.722050    0.702399    0.760638    0.742862    0.807247  ...   \n",
       "cg00000165    0.247898    0.286702    0.225935    0.269163    0.275686  ...   \n",
       "cg00000236    0.809988    0.796479    0.768113    0.773171    0.808128  ...   \n",
       "\n",
       "            GSM1443425  GSM1443711  GSM1443363  GSM1443400  GSM1443470  \\\n",
       "cg00000029    0.394504    0.469003    0.408238    0.485401    0.498065   \n",
       "cg00000108    0.905646    0.912011    0.915954    0.871444    0.899956   \n",
       "cg00000109    0.817154    0.823309    0.842378    0.827500    0.776130   \n",
       "cg00000165    0.241856    0.255883    0.214639    0.249356    0.256304   \n",
       "cg00000236    0.695515    0.707434    0.718788    0.727323    0.702707   \n",
       "\n",
       "            GSM1443372  GSM1443303  GSM1443617  GSM1443348  GSM1443322  \n",
       "cg00000029    0.554049    0.468078    0.470046    0.312732    0.396636  \n",
       "cg00000108    0.896206    0.903646    0.901356    0.913590    0.895599  \n",
       "cg00000109    0.829127    0.816067    0.839910    0.802812    0.800367  \n",
       "cg00000165    0.312047    0.263720    0.270058    0.248141    0.273269  \n",
       "cg00000236    0.679252    0.668877    0.708453    0.665294    0.699228  \n",
       "\n",
       "[5 rows x 531 columns]"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_path = Path.joinpath(cwd, project_dir, \"{}_betas.csv\".format(project_id))\n",
    "print(data_path)\n",
    "df_percent = pd.read_csv(data_path, skiprows=4, index_col=0, header=[0,1,2])\n",
    "df_percent.columns = df_percent.columns.get_level_values(1)\n",
    "print(df_percent.shape)\n",
    "df_percent.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "id": "df703063-346c-4d83-9f35-711cf398bada",
   "metadata": {},
   "outputs": [],
   "source": [
    "whole_blood = df_meta[df_meta['Comment [Sample_source_name]'].values == 'whole blood'].index\n",
    "brain = df_meta[df_meta['Comment [Sample_source_name]'].values != 'whole blood'].index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "84cbb31f-c963-4762-ba13-54e3c2bf4551",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Source Name</th>\n",
       "      <th>GSM1443800</th>\n",
       "      <th>GSM1443799</th>\n",
       "      <th>GSM1443798</th>\n",
       "      <th>GSM1443797</th>\n",
       "      <th>GSM1443796</th>\n",
       "      <th>GSM1443740</th>\n",
       "      <th>GSM1443736</th>\n",
       "      <th>GSM1443722</th>\n",
       "      <th>GSM1443721</th>\n",
       "      <th>GSM1443712</th>\n",
       "      <th>...</th>\n",
       "      <th>GSM1443305</th>\n",
       "      <th>GSM1443303</th>\n",
       "      <th>GSM1443294</th>\n",
       "      <th>GSM1443290</th>\n",
       "      <th>GSM1443284</th>\n",
       "      <th>GSM1443278</th>\n",
       "      <th>GSM1443272</th>\n",
       "      <th>GSM1443266</th>\n",
       "      <th>GSM1443253</th>\n",
       "      <th>GSM1443250</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cg00000029</th>\n",
       "      <td>0.400600</td>\n",
       "      <td>0.479665</td>\n",
       "      <td>0.436074</td>\n",
       "      <td>0.514366</td>\n",
       "      <td>0.362122</td>\n",
       "      <td>0.387852</td>\n",
       "      <td>0.319614</td>\n",
       "      <td>0.339262</td>\n",
       "      <td>0.462028</td>\n",
       "      <td>0.433819</td>\n",
       "      <td>...</td>\n",
       "      <td>0.516897</td>\n",
       "      <td>0.468078</td>\n",
       "      <td>0.470120</td>\n",
       "      <td>0.451150</td>\n",
       "      <td>0.440332</td>\n",
       "      <td>0.509242</td>\n",
       "      <td>0.395939</td>\n",
       "      <td>0.490861</td>\n",
       "      <td>0.391604</td>\n",
       "      <td>0.456345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00000108</th>\n",
       "      <td>0.900679</td>\n",
       "      <td>0.916497</td>\n",
       "      <td>0.912981</td>\n",
       "      <td>0.899821</td>\n",
       "      <td>0.897078</td>\n",
       "      <td>0.896803</td>\n",
       "      <td>0.884033</td>\n",
       "      <td>0.906172</td>\n",
       "      <td>0.911777</td>\n",
       "      <td>0.906897</td>\n",
       "      <td>...</td>\n",
       "      <td>0.916217</td>\n",
       "      <td>0.903646</td>\n",
       "      <td>0.901680</td>\n",
       "      <td>0.909644</td>\n",
       "      <td>0.904179</td>\n",
       "      <td>0.901467</td>\n",
       "      <td>0.872436</td>\n",
       "      <td>0.917257</td>\n",
       "      <td>0.905948</td>\n",
       "      <td>0.914092</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00000109</th>\n",
       "      <td>0.784745</td>\n",
       "      <td>0.816812</td>\n",
       "      <td>0.798041</td>\n",
       "      <td>0.811868</td>\n",
       "      <td>0.800570</td>\n",
       "      <td>0.775996</td>\n",
       "      <td>0.772566</td>\n",
       "      <td>0.802771</td>\n",
       "      <td>0.758300</td>\n",
       "      <td>0.790358</td>\n",
       "      <td>...</td>\n",
       "      <td>0.791145</td>\n",
       "      <td>0.816067</td>\n",
       "      <td>0.822716</td>\n",
       "      <td>0.817535</td>\n",
       "      <td>0.790922</td>\n",
       "      <td>0.823270</td>\n",
       "      <td>0.838168</td>\n",
       "      <td>0.830552</td>\n",
       "      <td>0.807491</td>\n",
       "      <td>0.812966</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00000165</th>\n",
       "      <td>0.256220</td>\n",
       "      <td>0.204601</td>\n",
       "      <td>0.213874</td>\n",
       "      <td>0.329249</td>\n",
       "      <td>0.238178</td>\n",
       "      <td>0.241095</td>\n",
       "      <td>0.329328</td>\n",
       "      <td>0.223555</td>\n",
       "      <td>0.252695</td>\n",
       "      <td>0.288909</td>\n",
       "      <td>...</td>\n",
       "      <td>0.331216</td>\n",
       "      <td>0.263720</td>\n",
       "      <td>0.317720</td>\n",
       "      <td>0.262182</td>\n",
       "      <td>0.325583</td>\n",
       "      <td>0.266612</td>\n",
       "      <td>0.240405</td>\n",
       "      <td>0.271603</td>\n",
       "      <td>0.237282</td>\n",
       "      <td>0.246701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00000236</th>\n",
       "      <td>0.729144</td>\n",
       "      <td>0.699488</td>\n",
       "      <td>0.681784</td>\n",
       "      <td>0.718873</td>\n",
       "      <td>0.672487</td>\n",
       "      <td>0.640142</td>\n",
       "      <td>0.690584</td>\n",
       "      <td>0.705214</td>\n",
       "      <td>0.689278</td>\n",
       "      <td>0.720270</td>\n",
       "      <td>...</td>\n",
       "      <td>0.687372</td>\n",
       "      <td>0.668877</td>\n",
       "      <td>0.752853</td>\n",
       "      <td>0.678321</td>\n",
       "      <td>0.704015</td>\n",
       "      <td>0.688254</td>\n",
       "      <td>0.643808</td>\n",
       "      <td>0.746305</td>\n",
       "      <td>0.724604</td>\n",
       "      <td>0.711554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rs9363764</th>\n",
       "      <td>0.576389</td>\n",
       "      <td>0.536499</td>\n",
       "      <td>0.083839</td>\n",
       "      <td>0.080110</td>\n",
       "      <td>0.591569</td>\n",
       "      <td>0.559672</td>\n",
       "      <td>0.103649</td>\n",
       "      <td>0.905340</td>\n",
       "      <td>0.901959</td>\n",
       "      <td>0.558879</td>\n",
       "      <td>...</td>\n",
       "      <td>0.567609</td>\n",
       "      <td>0.553931</td>\n",
       "      <td>0.510338</td>\n",
       "      <td>0.907276</td>\n",
       "      <td>0.915557</td>\n",
       "      <td>0.558164</td>\n",
       "      <td>0.567917</td>\n",
       "      <td>0.558261</td>\n",
       "      <td>0.923414</td>\n",
       "      <td>0.897087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rs939290</th>\n",
       "      <td>0.595559</td>\n",
       "      <td>0.084885</td>\n",
       "      <td>0.575084</td>\n",
       "      <td>0.913868</td>\n",
       "      <td>0.590491</td>\n",
       "      <td>0.601533</td>\n",
       "      <td>0.604600</td>\n",
       "      <td>0.585897</td>\n",
       "      <td>0.585848</td>\n",
       "      <td>0.902012</td>\n",
       "      <td>...</td>\n",
       "      <td>0.619900</td>\n",
       "      <td>0.917878</td>\n",
       "      <td>0.912463</td>\n",
       "      <td>0.917604</td>\n",
       "      <td>0.569319</td>\n",
       "      <td>0.561251</td>\n",
       "      <td>0.920941</td>\n",
       "      <td>0.569607</td>\n",
       "      <td>0.093943</td>\n",
       "      <td>0.559108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rs951295</th>\n",
       "      <td>0.924107</td>\n",
       "      <td>0.514987</td>\n",
       "      <td>0.927003</td>\n",
       "      <td>0.511728</td>\n",
       "      <td>0.152104</td>\n",
       "      <td>0.928043</td>\n",
       "      <td>0.138360</td>\n",
       "      <td>0.523280</td>\n",
       "      <td>0.538093</td>\n",
       "      <td>0.534714</td>\n",
       "      <td>...</td>\n",
       "      <td>0.099319</td>\n",
       "      <td>0.923505</td>\n",
       "      <td>0.499863</td>\n",
       "      <td>0.925335</td>\n",
       "      <td>0.525631</td>\n",
       "      <td>0.126296</td>\n",
       "      <td>0.528046</td>\n",
       "      <td>0.520803</td>\n",
       "      <td>0.534060</td>\n",
       "      <td>0.528199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rs966367</th>\n",
       "      <td>0.486946</td>\n",
       "      <td>0.100176</td>\n",
       "      <td>0.538077</td>\n",
       "      <td>0.486893</td>\n",
       "      <td>0.122075</td>\n",
       "      <td>0.830692</td>\n",
       "      <td>0.117114</td>\n",
       "      <td>0.512356</td>\n",
       "      <td>0.122853</td>\n",
       "      <td>0.850329</td>\n",
       "      <td>...</td>\n",
       "      <td>0.535995</td>\n",
       "      <td>0.505963</td>\n",
       "      <td>0.470540</td>\n",
       "      <td>0.538690</td>\n",
       "      <td>0.495690</td>\n",
       "      <td>0.500017</td>\n",
       "      <td>0.537379</td>\n",
       "      <td>0.100682</td>\n",
       "      <td>0.879050</td>\n",
       "      <td>0.097134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rs9839873</th>\n",
       "      <td>0.593906</td>\n",
       "      <td>0.601118</td>\n",
       "      <td>0.887603</td>\n",
       "      <td>0.897336</td>\n",
       "      <td>0.592086</td>\n",
       "      <td>0.860027</td>\n",
       "      <td>0.657235</td>\n",
       "      <td>0.864611</td>\n",
       "      <td>0.574175</td>\n",
       "      <td>0.880277</td>\n",
       "      <td>...</td>\n",
       "      <td>0.887356</td>\n",
       "      <td>0.878604</td>\n",
       "      <td>0.859585</td>\n",
       "      <td>0.881588</td>\n",
       "      <td>0.891001</td>\n",
       "      <td>0.879726</td>\n",
       "      <td>0.859762</td>\n",
       "      <td>0.863417</td>\n",
       "      <td>0.897942</td>\n",
       "      <td>0.879961</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>485577 rows × 80 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Source Name  GSM1443800  GSM1443799  GSM1443798  GSM1443797  GSM1443796  \\\n",
       "cg00000029     0.400600    0.479665    0.436074    0.514366    0.362122   \n",
       "cg00000108     0.900679    0.916497    0.912981    0.899821    0.897078   \n",
       "cg00000109     0.784745    0.816812    0.798041    0.811868    0.800570   \n",
       "cg00000165     0.256220    0.204601    0.213874    0.329249    0.238178   \n",
       "cg00000236     0.729144    0.699488    0.681784    0.718873    0.672487   \n",
       "...                 ...         ...         ...         ...         ...   \n",
       "rs9363764      0.576389    0.536499    0.083839    0.080110    0.591569   \n",
       "rs939290       0.595559    0.084885    0.575084    0.913868    0.590491   \n",
       "rs951295       0.924107    0.514987    0.927003    0.511728    0.152104   \n",
       "rs966367       0.486946    0.100176    0.538077    0.486893    0.122075   \n",
       "rs9839873      0.593906    0.601118    0.887603    0.897336    0.592086   \n",
       "\n",
       "Source Name  GSM1443740  GSM1443736  GSM1443722  GSM1443721  GSM1443712  ...  \\\n",
       "cg00000029     0.387852    0.319614    0.339262    0.462028    0.433819  ...   \n",
       "cg00000108     0.896803    0.884033    0.906172    0.911777    0.906897  ...   \n",
       "cg00000109     0.775996    0.772566    0.802771    0.758300    0.790358  ...   \n",
       "cg00000165     0.241095    0.329328    0.223555    0.252695    0.288909  ...   \n",
       "cg00000236     0.640142    0.690584    0.705214    0.689278    0.720270  ...   \n",
       "...                 ...         ...         ...         ...         ...  ...   \n",
       "rs9363764      0.559672    0.103649    0.905340    0.901959    0.558879  ...   \n",
       "rs939290       0.601533    0.604600    0.585897    0.585848    0.902012  ...   \n",
       "rs951295       0.928043    0.138360    0.523280    0.538093    0.534714  ...   \n",
       "rs966367       0.830692    0.117114    0.512356    0.122853    0.850329  ...   \n",
       "rs9839873      0.860027    0.657235    0.864611    0.574175    0.880277  ...   \n",
       "\n",
       "Source Name  GSM1443305  GSM1443303  GSM1443294  GSM1443290  GSM1443284  \\\n",
       "cg00000029     0.516897    0.468078    0.470120    0.451150    0.440332   \n",
       "cg00000108     0.916217    0.903646    0.901680    0.909644    0.904179   \n",
       "cg00000109     0.791145    0.816067    0.822716    0.817535    0.790922   \n",
       "cg00000165     0.331216    0.263720    0.317720    0.262182    0.325583   \n",
       "cg00000236     0.687372    0.668877    0.752853    0.678321    0.704015   \n",
       "...                 ...         ...         ...         ...         ...   \n",
       "rs9363764      0.567609    0.553931    0.510338    0.907276    0.915557   \n",
       "rs939290       0.619900    0.917878    0.912463    0.917604    0.569319   \n",
       "rs951295       0.099319    0.923505    0.499863    0.925335    0.525631   \n",
       "rs966367       0.535995    0.505963    0.470540    0.538690    0.495690   \n",
       "rs9839873      0.887356    0.878604    0.859585    0.881588    0.891001   \n",
       "\n",
       "Source Name  GSM1443278  GSM1443272  GSM1443266  GSM1443253  GSM1443250  \n",
       "cg00000029     0.509242    0.395939    0.490861    0.391604    0.456345  \n",
       "cg00000108     0.901467    0.872436    0.917257    0.905948    0.914092  \n",
       "cg00000109     0.823270    0.838168    0.830552    0.807491    0.812966  \n",
       "cg00000165     0.266612    0.240405    0.271603    0.237282    0.246701  \n",
       "cg00000236     0.688254    0.643808    0.746305    0.724604    0.711554  \n",
       "...                 ...         ...         ...         ...         ...  \n",
       "rs9363764      0.558164    0.567917    0.558261    0.923414    0.897087  \n",
       "rs939290       0.561251    0.920941    0.569607    0.093943    0.559108  \n",
       "rs951295       0.126296    0.528046    0.520803    0.534060    0.528199  \n",
       "rs966367       0.500017    0.537379    0.100682    0.879050    0.097134  \n",
       "rs9839873      0.879726    0.859762    0.863417    0.897942    0.879961  \n",
       "\n",
       "[485577 rows x 80 columns]"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "d23ebc49-6569-4ab5-a8ab-3a150033fd79",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Comment [Sample_description]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Comment [Sample_source_name]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Characteristics [Sex]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Characteristics [ad.disease.status]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.1' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number.1' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Characteristics [braak.stage]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Characteristics [organism]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.2' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number.2' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Characteristics [source tissue]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.3' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number.3' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Protocol REF' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.4' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Material Type' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Protocol REF.1' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.5' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Label' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Protocol REF.2' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.6' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Array Design REF' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.7' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Technology Type' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'FactorValue [AD.DISEASE.STATUS]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.8' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number.4' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'FactorValue [BRAAK.STAGE]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'FactorValue [SEX]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.9' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number.5' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'FactorValue [SOURCE TISSUE]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.10' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number.6' as categorical\n"
     ]
    }
   ],
   "source": [
    "ad = sc.AnnData(df_percent.loc[:,whole_blood])\n",
    "ad.var = pd.concat((ad.var, df_meta.loc[ad.var.index,:]), axis=1)\n",
    "ad.X = ad.X.astype(np.float16)\n",
    "ad.write_h5ad(Path.joinpath(project_dir, \"{}_whole_blood.h5ad\".format(project_id)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "67509663-e234-4afc-b1f5-5711310f736d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Comment [Sample_description]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Comment [Sample_source_name]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Characteristics [Sex]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Characteristics [ad.disease.status]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.1' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number.1' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Characteristics [braak.stage]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Characteristics [organism]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.2' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number.2' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Characteristics [source tissue]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.3' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number.3' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Protocol REF' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.4' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Material Type' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Protocol REF.1' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.5' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Label' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Protocol REF.2' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.6' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Array Design REF' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.7' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Technology Type' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'FactorValue [AD.DISEASE.STATUS]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.8' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number.4' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'FactorValue [BRAAK.STAGE]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'FactorValue [SEX]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.9' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number.5' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'FactorValue [SOURCE TISSUE]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.10' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number.6' as categorical\n"
     ]
    }
   ],
   "source": [
    "ad = sc.AnnData(df_percent.loc[:,brain])\n",
    "ad.var = pd.concat((ad.var, df_meta.loc[ad.var.index,:]), axis=1)\n",
    "ad.X = ad.X.astype(np.float16)\n",
    "ad.write_h5ad(Path.joinpath(project_dir, \"{}_brain.h5ad\".format(project_id)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efac8be5-d1c3-4381-bc74-e9c4c6e01488",
   "metadata": {},
   "outputs": [],
   "source": [
    "ad = sc.AnnData(df_percent)\n",
    "ad.var = pd.concat((ad.var, df_meta.loc[ad.var.index,:]), axis=1)\n",
    "ad.X = ad.X.astype(np.float16)\n",
    "ad.write_h5ad(Path.joinpath(project_dir, \"{}_brain.h5ad\".format(project_id)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "7464bb8e-bce6-490c-bfb7-58ec21bafd81",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Comment [Sample_description]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Comment [Sample_source_name]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Characteristics [Sex]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Characteristics [ad.disease.status]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.1' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number.1' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Characteristics [braak.stage]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Characteristics [organism]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.2' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number.2' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Characteristics [source tissue]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.3' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number.3' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Protocol REF' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.4' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Material Type' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Protocol REF.1' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.5' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Label' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Protocol REF.2' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.6' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Array Design REF' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.7' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Technology Type' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'FactorValue [AD.DISEASE.STATUS]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.8' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number.4' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'FactorValue [BRAAK.STAGE]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'FactorValue [SEX]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.9' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number.5' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'FactorValue [SOURCE TISSUE]' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Source REF.10' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Term Accession Number.6' as categorical\n"
     ]
    }
   ],
   "source": [
    "ad.write_h5ad(Path.joinpath(project_dir, \"{}.h5ad\".format(project_id)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "46d36ba1-95ce-4094-b29f-bfa79026a383",
   "metadata": {},
   "source": [
    "# Brain aging methylation (GSE74193)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "id": "48fa8312-0efd-42c3-8eb0-499467c37474",
   "metadata": {},
   "outputs": [],
   "source": [
    "cwd = Path.cwd()\n",
    "# cwd = Path('/Users/tarkhov/Dropbox/Harvard/Science/Church/')\n",
    "project_dir = Path.joinpath(cwd, Path('data/GSE74193/'))\n",
    "project_id = 'GSE74193'\n",
    "project_samples_dir = 'GSE74193_RAW'\n",
    "metainfo_path = 'phenotype_data_forGeo.csv'\n",
    "gl_metainfo_path = Path.joinpath(project_dir, metainfo_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "fbc41fbc-5d5f-4f00-8696-79f15a83ac3e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(675, 27)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BrNum</th>\n",
       "      <th>Chip</th>\n",
       "      <th>Sentrix_ID</th>\n",
       "      <th>Sentrix_Position</th>\n",
       "      <th>Plate</th>\n",
       "      <th>DropSample</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Race</th>\n",
       "      <th>Dx</th>\n",
       "      <th>Age</th>\n",
       "      <th>...</th>\n",
       "      <th>xMed</th>\n",
       "      <th>yMed</th>\n",
       "      <th>predictedSex</th>\n",
       "      <th>bestQC</th>\n",
       "      <th>ES</th>\n",
       "      <th>NPC</th>\n",
       "      <th>DA_NEURON</th>\n",
       "      <th>NeuN_pos</th>\n",
       "      <th>NeuN_neg</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SampleID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Sample1</th>\n",
       "      <td>Br1811</td>\n",
       "      <td>5815188028_R04C01</td>\n",
       "      <td>5815188028</td>\n",
       "      <td>R04C01</td>\n",
       "      <td>Lieber_104</td>\n",
       "      <td>False</td>\n",
       "      <td>F</td>\n",
       "      <td>AA</td>\n",
       "      <td>Control</td>\n",
       "      <td>-0.49863</td>\n",
       "      <td>...</td>\n",
       "      <td>13.956058</td>\n",
       "      <td>9.240789</td>\n",
       "      <td>F</td>\n",
       "      <td>False</td>\n",
       "      <td>0.139772</td>\n",
       "      <td>0.135177</td>\n",
       "      <td>0.065425</td>\n",
       "      <td>0.125416</td>\n",
       "      <td>0.366008</td>\n",
       "      <td>Sample1_Control</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample2</th>\n",
       "      <td>Br1811</td>\n",
       "      <td>6229009146_R02C01</td>\n",
       "      <td>6229009146</td>\n",
       "      <td>R02C01</td>\n",
       "      <td>Lieber_289</td>\n",
       "      <td>False</td>\n",
       "      <td>F</td>\n",
       "      <td>AA</td>\n",
       "      <td>Control</td>\n",
       "      <td>-0.49863</td>\n",
       "      <td>...</td>\n",
       "      <td>13.682117</td>\n",
       "      <td>9.018200</td>\n",
       "      <td>F</td>\n",
       "      <td>True</td>\n",
       "      <td>0.147870</td>\n",
       "      <td>0.202554</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.129817</td>\n",
       "      <td>0.357413</td>\n",
       "      <td>Sample2_Control</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample3</th>\n",
       "      <td>Br1942</td>\n",
       "      <td>5806484002_R01C02</td>\n",
       "      <td>5806484002</td>\n",
       "      <td>R01C02</td>\n",
       "      <td>Lieber_104</td>\n",
       "      <td>False</td>\n",
       "      <td>F</td>\n",
       "      <td>AA</td>\n",
       "      <td>Control</td>\n",
       "      <td>-0.49863</td>\n",
       "      <td>...</td>\n",
       "      <td>13.867230</td>\n",
       "      <td>8.896332</td>\n",
       "      <td>F</td>\n",
       "      <td>False</td>\n",
       "      <td>0.108076</td>\n",
       "      <td>0.197264</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.159891</td>\n",
       "      <td>0.344638</td>\n",
       "      <td>Sample3_Control</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample4</th>\n",
       "      <td>Br1942</td>\n",
       "      <td>6229009147_R04C01</td>\n",
       "      <td>6229009147</td>\n",
       "      <td>R04C01</td>\n",
       "      <td>Lieber_289</td>\n",
       "      <td>False</td>\n",
       "      <td>F</td>\n",
       "      <td>AA</td>\n",
       "      <td>Control</td>\n",
       "      <td>-0.49863</td>\n",
       "      <td>...</td>\n",
       "      <td>13.887316</td>\n",
       "      <td>9.151012</td>\n",
       "      <td>F</td>\n",
       "      <td>True</td>\n",
       "      <td>0.123939</td>\n",
       "      <td>0.193156</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.156491</td>\n",
       "      <td>0.350472</td>\n",
       "      <td>Sample4_Control</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample5</th>\n",
       "      <td>Br1808</td>\n",
       "      <td>5806484010_R02C02</td>\n",
       "      <td>5806484010</td>\n",
       "      <td>R02C02</td>\n",
       "      <td>Lieber_244</td>\n",
       "      <td>False</td>\n",
       "      <td>M</td>\n",
       "      <td>AA</td>\n",
       "      <td>Control</td>\n",
       "      <td>-0.49863</td>\n",
       "      <td>...</td>\n",
       "      <td>13.324321</td>\n",
       "      <td>13.409449</td>\n",
       "      <td>M</td>\n",
       "      <td>True</td>\n",
       "      <td>0.133305</td>\n",
       "      <td>0.192826</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.149536</td>\n",
       "      <td>0.355354</td>\n",
       "      <td>Sample5_Control</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           BrNum               Chip  Sentrix_ID Sentrix_Position       Plate  \\\n",
       "SampleID                                                                       \n",
       "Sample1   Br1811  5815188028_R04C01  5815188028           R04C01  Lieber_104   \n",
       "Sample2   Br1811  6229009146_R02C01  6229009146           R02C01  Lieber_289   \n",
       "Sample3   Br1942  5806484002_R01C02  5806484002           R01C02  Lieber_104   \n",
       "Sample4   Br1942  6229009147_R04C01  6229009147           R04C01  Lieber_289   \n",
       "Sample5   Br1808  5806484010_R02C02  5806484010           R02C02  Lieber_244   \n",
       "\n",
       "          DropSample Gender Race       Dx      Age  ...       xMed       yMed  \\\n",
       "SampleID                                            ...                         \n",
       "Sample1        False      F   AA  Control -0.49863  ...  13.956058   9.240789   \n",
       "Sample2        False      F   AA  Control -0.49863  ...  13.682117   9.018200   \n",
       "Sample3        False      F   AA  Control -0.49863  ...  13.867230   8.896332   \n",
       "Sample4        False      F   AA  Control -0.49863  ...  13.887316   9.151012   \n",
       "Sample5        False      M   AA  Control -0.49863  ...  13.324321  13.409449   \n",
       "\n",
       "          predictedSex  bestQC        ES       NPC  DA_NEURON  NeuN_pos  \\\n",
       "SampleID                                                                  \n",
       "Sample1              F   False  0.139772  0.135177   0.065425  0.125416   \n",
       "Sample2              F    True  0.147870  0.202554   0.000000  0.129817   \n",
       "Sample3              F   False  0.108076  0.197264   0.000000  0.159891   \n",
       "Sample4              F    True  0.123939  0.193156   0.000000  0.156491   \n",
       "Sample5              M    True  0.133305  0.192826   0.000000  0.149536   \n",
       "\n",
       "          NeuN_neg            title  \n",
       "SampleID                             \n",
       "Sample1   0.366008  Sample1_Control  \n",
       "Sample2   0.357413  Sample2_Control  \n",
       "Sample3   0.344638  Sample3_Control  \n",
       "Sample4   0.350472  Sample4_Control  \n",
       "Sample5   0.355354  Sample5_Control  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_meta = pd.read_csv(gl_metainfo_path, index_col=0)\n",
    "# df_meta['Source Name'] = df_meta['Source Name'].apply(lambda x: x.split(' ')[0])\n",
    "# df_meta.set_index('Source Name', inplace=True)\n",
    "print(df_meta.shape)\n",
    "df_meta.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "1a0c55fb-50a1-417e-9d8b-5c3f62a2f95f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/tarkhov/Dropbox/Harvard/Science/Stochastic aging/data/GSE74193/GSE74193_GEO_procData.csv\n",
      "(485512, 675)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sample1</th>\n",
       "      <th>Sample2</th>\n",
       "      <th>Sample3</th>\n",
       "      <th>Sample4</th>\n",
       "      <th>Sample5</th>\n",
       "      <th>Sample6</th>\n",
       "      <th>Sample7</th>\n",
       "      <th>Sample8</th>\n",
       "      <th>Sample9</th>\n",
       "      <th>Sample10</th>\n",
       "      <th>...</th>\n",
       "      <th>Sample666</th>\n",
       "      <th>Sample667</th>\n",
       "      <th>Sample668</th>\n",
       "      <th>Sample669</th>\n",
       "      <th>Sample670</th>\n",
       "      <th>Sample671</th>\n",
       "      <th>Sample672</th>\n",
       "      <th>Sample673</th>\n",
       "      <th>Sample674</th>\n",
       "      <th>Sample675</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cg13869341</th>\n",
       "      <td>0.899729</td>\n",
       "      <td>0.900024</td>\n",
       "      <td>0.860819</td>\n",
       "      <td>0.869993</td>\n",
       "      <td>0.882916</td>\n",
       "      <td>0.896152</td>\n",
       "      <td>0.885027</td>\n",
       "      <td>0.863326</td>\n",
       "      <td>0.889406</td>\n",
       "      <td>0.892016</td>\n",
       "      <td>...</td>\n",
       "      <td>0.896810</td>\n",
       "      <td>0.888805</td>\n",
       "      <td>0.899020</td>\n",
       "      <td>0.845374</td>\n",
       "      <td>0.848777</td>\n",
       "      <td>0.885100</td>\n",
       "      <td>0.866056</td>\n",
       "      <td>0.875163</td>\n",
       "      <td>0.894426</td>\n",
       "      <td>0.878247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg14008030</th>\n",
       "      <td>0.672284</td>\n",
       "      <td>0.690021</td>\n",
       "      <td>0.732351</td>\n",
       "      <td>0.745440</td>\n",
       "      <td>0.776872</td>\n",
       "      <td>0.818895</td>\n",
       "      <td>0.696498</td>\n",
       "      <td>0.677681</td>\n",
       "      <td>0.761379</td>\n",
       "      <td>0.692668</td>\n",
       "      <td>...</td>\n",
       "      <td>0.733004</td>\n",
       "      <td>0.749770</td>\n",
       "      <td>0.746412</td>\n",
       "      <td>0.675795</td>\n",
       "      <td>0.713369</td>\n",
       "      <td>0.693225</td>\n",
       "      <td>0.675816</td>\n",
       "      <td>0.713268</td>\n",
       "      <td>0.667796</td>\n",
       "      <td>0.736118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg12045430</th>\n",
       "      <td>0.173613</td>\n",
       "      <td>0.269119</td>\n",
       "      <td>0.125225</td>\n",
       "      <td>0.167448</td>\n",
       "      <td>0.218392</td>\n",
       "      <td>0.237535</td>\n",
       "      <td>0.219908</td>\n",
       "      <td>0.177685</td>\n",
       "      <td>0.099079</td>\n",
       "      <td>0.117321</td>\n",
       "      <td>...</td>\n",
       "      <td>0.113756</td>\n",
       "      <td>0.226086</td>\n",
       "      <td>0.156269</td>\n",
       "      <td>0.116822</td>\n",
       "      <td>0.123014</td>\n",
       "      <td>0.190331</td>\n",
       "      <td>0.152175</td>\n",
       "      <td>0.116098</td>\n",
       "      <td>0.130616</td>\n",
       "      <td>0.142643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg20826792</th>\n",
       "      <td>0.459919</td>\n",
       "      <td>0.250206</td>\n",
       "      <td>0.380940</td>\n",
       "      <td>0.275163</td>\n",
       "      <td>0.445132</td>\n",
       "      <td>0.429192</td>\n",
       "      <td>0.210046</td>\n",
       "      <td>0.433255</td>\n",
       "      <td>0.222208</td>\n",
       "      <td>0.413142</td>\n",
       "      <td>...</td>\n",
       "      <td>0.409681</td>\n",
       "      <td>0.203996</td>\n",
       "      <td>0.415888</td>\n",
       "      <td>0.332559</td>\n",
       "      <td>0.400732</td>\n",
       "      <td>0.247789</td>\n",
       "      <td>0.438822</td>\n",
       "      <td>0.356353</td>\n",
       "      <td>0.308095</td>\n",
       "      <td>0.219863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00381604</th>\n",
       "      <td>0.156861</td>\n",
       "      <td>0.307937</td>\n",
       "      <td>0.175469</td>\n",
       "      <td>0.097830</td>\n",
       "      <td>0.187581</td>\n",
       "      <td>0.250647</td>\n",
       "      <td>0.081984</td>\n",
       "      <td>0.152077</td>\n",
       "      <td>0.178939</td>\n",
       "      <td>0.137189</td>\n",
       "      <td>...</td>\n",
       "      <td>0.159757</td>\n",
       "      <td>0.098771</td>\n",
       "      <td>0.152755</td>\n",
       "      <td>0.124415</td>\n",
       "      <td>0.146928</td>\n",
       "      <td>0.060124</td>\n",
       "      <td>0.150322</td>\n",
       "      <td>0.122493</td>\n",
       "      <td>0.135896</td>\n",
       "      <td>0.227564</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 675 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Sample1   Sample2   Sample3   Sample4   Sample5   Sample6  \\\n",
       "cg13869341  0.899729  0.900024  0.860819  0.869993  0.882916  0.896152   \n",
       "cg14008030  0.672284  0.690021  0.732351  0.745440  0.776872  0.818895   \n",
       "cg12045430  0.173613  0.269119  0.125225  0.167448  0.218392  0.237535   \n",
       "cg20826792  0.459919  0.250206  0.380940  0.275163  0.445132  0.429192   \n",
       "cg00381604  0.156861  0.307937  0.175469  0.097830  0.187581  0.250647   \n",
       "\n",
       "             Sample7   Sample8   Sample9  Sample10  ...  Sample666  Sample667  \\\n",
       "cg13869341  0.885027  0.863326  0.889406  0.892016  ...   0.896810   0.888805   \n",
       "cg14008030  0.696498  0.677681  0.761379  0.692668  ...   0.733004   0.749770   \n",
       "cg12045430  0.219908  0.177685  0.099079  0.117321  ...   0.113756   0.226086   \n",
       "cg20826792  0.210046  0.433255  0.222208  0.413142  ...   0.409681   0.203996   \n",
       "cg00381604  0.081984  0.152077  0.178939  0.137189  ...   0.159757   0.098771   \n",
       "\n",
       "            Sample668  Sample669  Sample670  Sample671  Sample672  Sample673  \\\n",
       "cg13869341   0.899020   0.845374   0.848777   0.885100   0.866056   0.875163   \n",
       "cg14008030   0.746412   0.675795   0.713369   0.693225   0.675816   0.713268   \n",
       "cg12045430   0.156269   0.116822   0.123014   0.190331   0.152175   0.116098   \n",
       "cg20826792   0.415888   0.332559   0.400732   0.247789   0.438822   0.356353   \n",
       "cg00381604   0.152755   0.124415   0.146928   0.060124   0.150322   0.122493   \n",
       "\n",
       "            Sample674  Sample675  \n",
       "cg13869341   0.894426   0.878247  \n",
       "cg14008030   0.667796   0.736118  \n",
       "cg12045430   0.130616   0.142643  \n",
       "cg20826792   0.308095   0.219863  \n",
       "cg00381604   0.135896   0.227564  \n",
       "\n",
       "[5 rows x 675 columns]"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_path = Path.joinpath(cwd, project_dir, \"{}_GEO_procData.csv\".format(project_id))\n",
    "print(data_path)\n",
    "cols = pd.read_csv(data_path, nrows=0).columns\n",
    "df_percent = pd.read_csv(data_path, index_col=0, usecols=[cols[0]] + cols[1::2].tolist())\n",
    "df_percent.columns = [x.split('_')[0] for x in df_percent.columns]\n",
    "# df_percent = mtools.get_methylation_df(data_path, typ='percents')\n",
    "# df_percent.columns = df_percent.columns.get_level_values(1)\n",
    "print(df_percent.shape)\n",
    "df_percent.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "267e3b4b-4bd3-491c-84fd-2dda727a013e",
   "metadata": {},
   "outputs": [],
   "source": [
    "ad = sc.AnnData(df_percent)\n",
    "ad.var = pd.concat((ad.var, df_meta.loc[ad.var.index,:]), axis=1)\n",
    "ad.X = ad.X.astype(np.float16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "5c11456c-4ed5-44bb-828d-81789fe9be09",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'BrNum' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Sentrix_Position' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Plate' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Gender' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Race' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Dx' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'ageGroup' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'predictedSex' as categorical\n"
     ]
    }
   ],
   "source": [
    "ad.write_h5ad(Path.joinpath(project_dir, \"{}.h5ad\".format(project_id)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c453870-cbbb-4359-a4dd-7aa5847b1102",
   "metadata": {},
   "source": [
    "# Brain reprogrammed HAF -> iNeurons (GSE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "7e7854d2-c4a6-4712-8661-a1081ca5df6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "cwd = Path.cwd()\n",
    "# cwd = Path('/Users/tarkhov/Dropbox/Harvard/Science/Church/')\n",
    "project_dir = Path.joinpath(cwd, Path('data/elife18648'))\n",
    "project_id = 'elife18648'\n",
    "project_samples_dir = 'elife18648_RAW'\n",
    "metainfo_path = 'DNAmethylation/datSampleNoobUpload.csv'\n",
    "gl_metainfo_path = Path.joinpath(project_dir, metainfo_path)\n",
    "metainfo_paired_path = 'DNAmethylation/datPairedData.csv'\n",
    "gl_metainfo_paired_path = Path.joinpath(project_dir, metainfo_paired_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "8362445b-c0db-4cd1-a96c-43122a8a2f97",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(53, 6)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DNAmAge</th>\n",
       "      <th>ExternalSampleID</th>\n",
       "      <th>Tissue</th>\n",
       "      <th>Age</th>\n",
       "      <th>Gender</th>\n",
       "      <th>TransformationStatus</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SampleNo</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Sample1</th>\n",
       "      <td>43.430715</td>\n",
       "      <td>17B-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>17.0</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample2</th>\n",
       "      <td>47.061787</td>\n",
       "      <td>72-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>72.0</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample3</th>\n",
       "      <td>36.181183</td>\n",
       "      <td>01-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample4</th>\n",
       "      <td>32.291262</td>\n",
       "      <td>01-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>1.0</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample5</th>\n",
       "      <td>46.297813</td>\n",
       "      <td>44-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>44.0</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            DNAmAge ExternalSampleID       Tissue   Age Gender  \\\n",
       "SampleNo                                                         \n",
       "Sample1   43.430715            17B-C  Fibroblasts  17.0   male   \n",
       "Sample2   47.061787             72-C  Fibroblasts  72.0   male   \n",
       "Sample3   36.181183             01-M          MSN   1.0   male   \n",
       "Sample4   32.291262             01-C  Fibroblasts   1.0   male   \n",
       "Sample5   46.297813             44-M          MSN  44.0   male   \n",
       "\n",
       "                TransformationStatus  \n",
       "SampleNo                              \n",
       "Sample1                Untransformed  \n",
       "Sample2                Untransformed  \n",
       "Sample3   Lentiviral miR-9/9-124-CDM  \n",
       "Sample4                Untransformed  \n",
       "Sample5   Lentiviral miR-9/9-124-CDM  "
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_meta = pd.read_csv(gl_metainfo_path, index_col=0)\n",
    "print(df_meta.shape)\n",
    "df_meta.reset_index(inplace=True)\n",
    "df_meta['SampleNo'] = df_meta['SampleNo'].apply(lambda x: ''.join(x.split('No')))\n",
    "df_meta.set_index('SampleNo', inplace=True)\n",
    "df_meta.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "0a3936f9-fe42-473a-83be-9414c5468611",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DNAmAge</th>\n",
       "      <th>ExternalSampleID</th>\n",
       "      <th>Tissue</th>\n",
       "      <th>Age</th>\n",
       "      <th>Gender</th>\n",
       "      <th>TransformationStatus</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SampleNo</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Sample1</th>\n",
       "      <td>43.430715</td>\n",
       "      <td>17B-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>17.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample2</th>\n",
       "      <td>47.061787</td>\n",
       "      <td>72-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>72.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample3</th>\n",
       "      <td>36.181183</td>\n",
       "      <td>01-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>1.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample4</th>\n",
       "      <td>32.291262</td>\n",
       "      <td>01-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>1.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample5</th>\n",
       "      <td>46.297813</td>\n",
       "      <td>44-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>44.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample6</th>\n",
       "      <td>45.762984</td>\n",
       "      <td>44-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>44.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample7</th>\n",
       "      <td>64.561971</td>\n",
       "      <td>92A-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>92.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample8</th>\n",
       "      <td>71.634253</td>\n",
       "      <td>92A-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>92.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample9</th>\n",
       "      <td>51.734553</td>\n",
       "      <td>92B-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>92.00</td>\n",
       "      <td>female</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample10</th>\n",
       "      <td>45.524989</td>\n",
       "      <td>31-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>31.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample11</th>\n",
       "      <td>40.024259</td>\n",
       "      <td>31-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>31.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample12</th>\n",
       "      <td>36.185683</td>\n",
       "      <td>10B-C-2</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>10.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample13</th>\n",
       "      <td>55.606625</td>\n",
       "      <td>11A-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>11.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample14</th>\n",
       "      <td>31.185213</td>\n",
       "      <td>17A-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>17.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample15</th>\n",
       "      <td>38.076881</td>\n",
       "      <td>05M-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>0.42</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample16</th>\n",
       "      <td>26.929950</td>\n",
       "      <td>05M-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>0.42</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample17</th>\n",
       "      <td>23.461219</td>\n",
       "      <td>07B-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>7.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample18</th>\n",
       "      <td>37.497850</td>\n",
       "      <td>08A-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>8.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample19</th>\n",
       "      <td>66.134065</td>\n",
       "      <td>92A-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>92.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample20</th>\n",
       "      <td>40.591273</td>\n",
       "      <td>90-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>90.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample21</th>\n",
       "      <td>21.565663</td>\n",
       "      <td>05-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>5.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample22</th>\n",
       "      <td>43.430629</td>\n",
       "      <td>05-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>5.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample23</th>\n",
       "      <td>64.544543</td>\n",
       "      <td>56-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>56.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample24</th>\n",
       "      <td>38.302485</td>\n",
       "      <td>10C-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>10.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample25</th>\n",
       "      <td>37.274027</td>\n",
       "      <td>10A-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>10.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample26</th>\n",
       "      <td>31.594701</td>\n",
       "      <td>08B-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>8.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample27</th>\n",
       "      <td>38.342321</td>\n",
       "      <td>10B-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>10.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample28</th>\n",
       "      <td>39.376480</td>\n",
       "      <td>07A-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>7.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample29</th>\n",
       "      <td>41.468048</td>\n",
       "      <td>90-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>90.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample30</th>\n",
       "      <td>65.373428</td>\n",
       "      <td>89-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>89.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample31</th>\n",
       "      <td>41.425684</td>\n",
       "      <td>90-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>90.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample32</th>\n",
       "      <td>62.312304</td>\n",
       "      <td>89-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>89.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample33</th>\n",
       "      <td>17.071664</td>\n",
       "      <td>03D-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>0.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample34</th>\n",
       "      <td>19.473595</td>\n",
       "      <td>03D-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>0.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample35</th>\n",
       "      <td>55.454335</td>\n",
       "      <td>96-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>96.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample36</th>\n",
       "      <td>81.034239</td>\n",
       "      <td>91-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>91.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample37</th>\n",
       "      <td>47.888099</td>\n",
       "      <td>92B-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>92.00</td>\n",
       "      <td>female</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample38</th>\n",
       "      <td>82.838960</td>\n",
       "      <td>91-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>91.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample39</th>\n",
       "      <td>17.769244</td>\n",
       "      <td>02-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>2.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample40</th>\n",
       "      <td>23.509896</td>\n",
       "      <td>02-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>2.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample41</th>\n",
       "      <td>41.505508</td>\n",
       "      <td>12-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>12.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample42</th>\n",
       "      <td>41.604927</td>\n",
       "      <td>12-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>12.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample43</th>\n",
       "      <td>41.846104</td>\n",
       "      <td>12B-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>12.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample44</th>\n",
       "      <td>32.252920</td>\n",
       "      <td>06-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>6.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample45</th>\n",
       "      <td>35.196626</td>\n",
       "      <td>12B-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>12.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample46</th>\n",
       "      <td>50.058842</td>\n",
       "      <td>19A-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>19.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample47</th>\n",
       "      <td>39.463079</td>\n",
       "      <td>19C-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>19.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample48</th>\n",
       "      <td>60.286239</td>\n",
       "      <td>68-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>68.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample49</th>\n",
       "      <td>65.275427</td>\n",
       "      <td>68-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>68.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample50</th>\n",
       "      <td>58.554913</td>\n",
       "      <td>94-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>94.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample51</th>\n",
       "      <td>56.334783</td>\n",
       "      <td>74-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>74.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample52</th>\n",
       "      <td>56.810323</td>\n",
       "      <td>94-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>94.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample53</th>\n",
       "      <td>55.448769</td>\n",
       "      <td>86-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>86.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            DNAmAge ExternalSampleID       Tissue    Age  Gender  \\\n",
       "SampleNo                                                           \n",
       "Sample1   43.430715            17B-C  Fibroblasts  17.00    male   \n",
       "Sample2   47.061787             72-C  Fibroblasts  72.00    male   \n",
       "Sample3   36.181183             01-M          MSN   1.00    male   \n",
       "Sample4   32.291262             01-C  Fibroblasts   1.00    male   \n",
       "Sample5   46.297813             44-M          MSN  44.00    male   \n",
       "Sample6   45.762984             44-C  Fibroblasts  44.00    male   \n",
       "Sample7   64.561971            92A-M          MSN  92.00    male   \n",
       "Sample8   71.634253            92A-C  Fibroblasts  92.00    male   \n",
       "Sample9   51.734553            92B-C  Fibroblasts  92.00  female   \n",
       "Sample10  45.524989             31-M          MSN  31.00    male   \n",
       "Sample11  40.024259             31-C  Fibroblasts  31.00    male   \n",
       "Sample12  36.185683          10B-C-2  Fibroblasts  10.00    male   \n",
       "Sample13  55.606625            11A-C  Fibroblasts  11.00    male   \n",
       "Sample14  31.185213            17A-C  Fibroblasts  17.00    male   \n",
       "Sample15  38.076881            05M-M          MSN   0.42    male   \n",
       "Sample16  26.929950            05M-C  Fibroblasts   0.42    male   \n",
       "Sample17  23.461219            07B-C  Fibroblasts   7.00    male   \n",
       "Sample18  37.497850            08A-C  Fibroblasts   8.00    male   \n",
       "Sample19  66.134065            92A-C  Fibroblasts  92.00    male   \n",
       "Sample20  40.591273             90-C  Fibroblasts  90.00    male   \n",
       "Sample21  21.565663             05-M          MSN   5.00    male   \n",
       "Sample22  43.430629             05-C  Fibroblasts   5.00    male   \n",
       "Sample23  64.544543             56-C  Fibroblasts  56.00    male   \n",
       "Sample24  38.302485            10C-C  Fibroblasts  10.00    male   \n",
       "Sample25  37.274027            10A-C  Fibroblasts  10.00    male   \n",
       "Sample26  31.594701            08B-C  Fibroblasts   8.00    male   \n",
       "Sample27  38.342321            10B-C  Fibroblasts  10.00    male   \n",
       "Sample28  39.376480            07A-C  Fibroblasts   7.00    male   \n",
       "Sample29  41.468048             90-M          MSN  90.00    male   \n",
       "Sample30  65.373428             89-M          MSN  89.00    male   \n",
       "Sample31  41.425684             90-C  Fibroblasts  90.00    male   \n",
       "Sample32  62.312304             89-C  Fibroblasts  89.00    male   \n",
       "Sample33  17.071664            03D-M          MSN   0.00    male   \n",
       "Sample34  19.473595            03D-C  Fibroblasts   0.00    male   \n",
       "Sample35  55.454335             96-M          MSN  96.00    male   \n",
       "Sample36  81.034239             91-M          MSN  91.00    male   \n",
       "Sample37  47.888099            92B-C  Fibroblasts  92.00  female   \n",
       "Sample38  82.838960             91-C  Fibroblasts  91.00    male   \n",
       "Sample39  17.769244             02-M          MSN   2.00    male   \n",
       "Sample40  23.509896             02-C  Fibroblasts   2.00    male   \n",
       "Sample41  41.505508             12-M          MSN  12.00    male   \n",
       "Sample42  41.604927             12-C  Fibroblasts  12.00    male   \n",
       "Sample43  41.846104            12B-M          MSN  12.00    male   \n",
       "Sample44  32.252920             06-C  Fibroblasts   6.00    male   \n",
       "Sample45  35.196626            12B-C  Fibroblasts  12.00    male   \n",
       "Sample46  50.058842            19A-C  Fibroblasts  19.00    male   \n",
       "Sample47  39.463079            19C-C  Fibroblasts  19.00    male   \n",
       "Sample48  60.286239             68-M          MSN  68.00    male   \n",
       "Sample49  65.275427             68-C  Fibroblasts  68.00    male   \n",
       "Sample50  58.554913             94-M          MSN  94.00    male   \n",
       "Sample51  56.334783             74-C  Fibroblasts  74.00    male   \n",
       "Sample52  56.810323             94-C  Fibroblasts  94.00    male   \n",
       "Sample53  55.448769             86-C  Fibroblasts  86.00    male   \n",
       "\n",
       "                TransformationStatus  \n",
       "SampleNo                              \n",
       "Sample1                Untransformed  \n",
       "Sample2                Untransformed  \n",
       "Sample3   Lentiviral miR-9/9-124-CDM  \n",
       "Sample4                Untransformed  \n",
       "Sample5   Lentiviral miR-9/9-124-CDM  \n",
       "Sample6                Untransformed  \n",
       "Sample7   Lentiviral miR-9/9-124-CDM  \n",
       "Sample8                Untransformed  \n",
       "Sample9                Untransformed  \n",
       "Sample10  Lentiviral miR-9/9-124-CDM  \n",
       "Sample11               Untransformed  \n",
       "Sample12               Untransformed  \n",
       "Sample13               Untransformed  \n",
       "Sample14               Untransformed  \n",
       "Sample15  Lentiviral miR-9/9-124-CDM  \n",
       "Sample16               Untransformed  \n",
       "Sample17               Untransformed  \n",
       "Sample18               Untransformed  \n",
       "Sample19               Untransformed  \n",
       "Sample20               Untransformed  \n",
       "Sample21  Lentiviral miR-9/9-124-CDM  \n",
       "Sample22               Untransformed  \n",
       "Sample23               Untransformed  \n",
       "Sample24               Untransformed  \n",
       "Sample25               Untransformed  \n",
       "Sample26               Untransformed  \n",
       "Sample27               Untransformed  \n",
       "Sample28               Untransformed  \n",
       "Sample29  Lentiviral miR-9/9-124-CDM  \n",
       "Sample30  Lentiviral miR-9/9-124-CDM  \n",
       "Sample31               Untransformed  \n",
       "Sample32               Untransformed  \n",
       "Sample33  Lentiviral miR-9/9-124-CDM  \n",
       "Sample34               Untransformed  \n",
       "Sample35  Lentiviral miR-9/9-124-CDM  \n",
       "Sample36  Lentiviral miR-9/9-124-CDM  \n",
       "Sample37               Untransformed  \n",
       "Sample38               Untransformed  \n",
       "Sample39  Lentiviral miR-9/9-124-CDM  \n",
       "Sample40               Untransformed  \n",
       "Sample41  Lentiviral miR-9/9-124-CDM  \n",
       "Sample42               Untransformed  \n",
       "Sample43  Lentiviral miR-9/9-124-CDM  \n",
       "Sample44               Untransformed  \n",
       "Sample45               Untransformed  \n",
       "Sample46               Untransformed  \n",
       "Sample47               Untransformed  \n",
       "Sample48  Lentiviral miR-9/9-124-CDM  \n",
       "Sample49               Untransformed  \n",
       "Sample50  Lentiviral miR-9/9-124-CDM  \n",
       "Sample51               Untransformed  \n",
       "Sample52               Untransformed  \n",
       "Sample53               Untransformed  "
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "2111a868-96ba-4dd2-9330-9adc2d80f7a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ExternalSampleIDFibroblast</th>\n",
       "      <th>ExternalSampleIDMSN</th>\n",
       "      <th>MatchingPairHorvath</th>\n",
       "      <th>Age</th>\n",
       "      <th>DNAmAgeFibroblast</th>\n",
       "      <th>DNAmAgeMSN</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01-C</td>\n",
       "      <td>01-M</td>\n",
       "      <td>1</td>\n",
       "      <td>1.00</td>\n",
       "      <td>32.291262</td>\n",
       "      <td>36.181183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>44-C</td>\n",
       "      <td>44-M</td>\n",
       "      <td>9</td>\n",
       "      <td>44.00</td>\n",
       "      <td>45.762984</td>\n",
       "      <td>46.297813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>92A-C</td>\n",
       "      <td>92A-M</td>\n",
       "      <td>14</td>\n",
       "      <td>92.00</td>\n",
       "      <td>71.634253</td>\n",
       "      <td>64.561971</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>31-C</td>\n",
       "      <td>31-M</td>\n",
       "      <td>8</td>\n",
       "      <td>31.00</td>\n",
       "      <td>40.024259</td>\n",
       "      <td>45.524989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>05M-C</td>\n",
       "      <td>05M-M</td>\n",
       "      <td>5</td>\n",
       "      <td>0.42</td>\n",
       "      <td>26.929950</td>\n",
       "      <td>38.076881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>05-C</td>\n",
       "      <td>05-M</td>\n",
       "      <td>4</td>\n",
       "      <td>5.00</td>\n",
       "      <td>43.430629</td>\n",
       "      <td>21.565663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>90-C</td>\n",
       "      <td>90-M</td>\n",
       "      <td>12</td>\n",
       "      <td>90.00</td>\n",
       "      <td>41.425684</td>\n",
       "      <td>41.468048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>89-C</td>\n",
       "      <td>89-M</td>\n",
       "      <td>11</td>\n",
       "      <td>89.00</td>\n",
       "      <td>62.312304</td>\n",
       "      <td>65.373428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>03D-C</td>\n",
       "      <td>03D-M</td>\n",
       "      <td>3</td>\n",
       "      <td>0.00</td>\n",
       "      <td>19.473595</td>\n",
       "      <td>17.071664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>91-C</td>\n",
       "      <td>91-M</td>\n",
       "      <td>13</td>\n",
       "      <td>91.00</td>\n",
       "      <td>82.838960</td>\n",
       "      <td>81.034239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>02-C</td>\n",
       "      <td>02-M</td>\n",
       "      <td>2</td>\n",
       "      <td>2.00</td>\n",
       "      <td>23.509896</td>\n",
       "      <td>17.769244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12-C</td>\n",
       "      <td>12-M</td>\n",
       "      <td>7</td>\n",
       "      <td>12.00</td>\n",
       "      <td>41.604927</td>\n",
       "      <td>41.505508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>12B-C</td>\n",
       "      <td>12B-M</td>\n",
       "      <td>6</td>\n",
       "      <td>12.00</td>\n",
       "      <td>35.196626</td>\n",
       "      <td>41.846104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>68-C</td>\n",
       "      <td>68-M</td>\n",
       "      <td>10</td>\n",
       "      <td>68.00</td>\n",
       "      <td>65.275427</td>\n",
       "      <td>60.286239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>94-C</td>\n",
       "      <td>94-M</td>\n",
       "      <td>15</td>\n",
       "      <td>94.00</td>\n",
       "      <td>56.810323</td>\n",
       "      <td>58.554913</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ExternalSampleIDFibroblast ExternalSampleIDMSN  MatchingPairHorvath    Age  \\\n",
       "0                        01-C                01-M                    1   1.00   \n",
       "1                        44-C                44-M                    9  44.00   \n",
       "2                       92A-C               92A-M                   14  92.00   \n",
       "3                        31-C                31-M                    8  31.00   \n",
       "4                       05M-C               05M-M                    5   0.42   \n",
       "5                        05-C                05-M                    4   5.00   \n",
       "6                        90-C                90-M                   12  90.00   \n",
       "7                        89-C                89-M                   11  89.00   \n",
       "8                       03D-C               03D-M                    3   0.00   \n",
       "9                        91-C                91-M                   13  91.00   \n",
       "10                       02-C                02-M                    2   2.00   \n",
       "11                       12-C                12-M                    7  12.00   \n",
       "12                      12B-C               12B-M                    6  12.00   \n",
       "13                       68-C                68-M                   10  68.00   \n",
       "14                       94-C                94-M                   15  94.00   \n",
       "\n",
       "    DNAmAgeFibroblast  DNAmAgeMSN  \n",
       "0           32.291262   36.181183  \n",
       "1           45.762984   46.297813  \n",
       "2           71.634253   64.561971  \n",
       "3           40.024259   45.524989  \n",
       "4           26.929950   38.076881  \n",
       "5           43.430629   21.565663  \n",
       "6           41.425684   41.468048  \n",
       "7           62.312304   65.373428  \n",
       "8           19.473595   17.071664  \n",
       "9           82.838960   81.034239  \n",
       "10          23.509896   17.769244  \n",
       "11          41.604927   41.505508  \n",
       "12          35.196626   41.846104  \n",
       "13          65.275427   60.286239  \n",
       "14          56.810323   58.554913  "
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_meta_paired = pd.read_csv(gl_metainfo_paired_path)\n",
    "id2pair = dict(zip(np.hstack((df_meta_paired.ExternalSampleIDFibroblast.values, df_meta_paired.ExternalSampleIDMSN.values)), np.hstack((df_meta_paired.MatchingPairHorvath.values,df_meta_paired.MatchingPairHorvath.values))))\n",
    "df_meta_paired#.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "e7473b22-8861-4330-aa20-b3f7f6f0e2a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DNAmAge</th>\n",
       "      <th>ExternalSampleID</th>\n",
       "      <th>Tissue</th>\n",
       "      <th>Age</th>\n",
       "      <th>Gender</th>\n",
       "      <th>TransformationStatus</th>\n",
       "      <th>MatchingPairHorvath</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SampleNo</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Sample1</th>\n",
       "      <td>43.430715</td>\n",
       "      <td>17B-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>17.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample2</th>\n",
       "      <td>47.061787</td>\n",
       "      <td>72-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>72.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample3</th>\n",
       "      <td>36.181183</td>\n",
       "      <td>01-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>1.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample4</th>\n",
       "      <td>32.291262</td>\n",
       "      <td>01-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>1.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample5</th>\n",
       "      <td>46.297813</td>\n",
       "      <td>44-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>44.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample6</th>\n",
       "      <td>45.762984</td>\n",
       "      <td>44-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>44.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample7</th>\n",
       "      <td>64.561971</td>\n",
       "      <td>92A-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>92.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample8</th>\n",
       "      <td>71.634253</td>\n",
       "      <td>92A-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>92.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample9</th>\n",
       "      <td>51.734553</td>\n",
       "      <td>92B-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>92.00</td>\n",
       "      <td>female</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample10</th>\n",
       "      <td>45.524989</td>\n",
       "      <td>31-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>31.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample11</th>\n",
       "      <td>40.024259</td>\n",
       "      <td>31-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>31.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample12</th>\n",
       "      <td>36.185683</td>\n",
       "      <td>10B-C-2</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>10.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample13</th>\n",
       "      <td>55.606625</td>\n",
       "      <td>11A-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>11.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample14</th>\n",
       "      <td>31.185213</td>\n",
       "      <td>17A-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>17.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample15</th>\n",
       "      <td>38.076881</td>\n",
       "      <td>05M-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>0.42</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample16</th>\n",
       "      <td>26.929950</td>\n",
       "      <td>05M-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>0.42</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample17</th>\n",
       "      <td>23.461219</td>\n",
       "      <td>07B-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>7.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample18</th>\n",
       "      <td>37.497850</td>\n",
       "      <td>08A-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>8.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample19</th>\n",
       "      <td>66.134065</td>\n",
       "      <td>92A-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>92.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample20</th>\n",
       "      <td>40.591273</td>\n",
       "      <td>90-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>90.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample21</th>\n",
       "      <td>21.565663</td>\n",
       "      <td>05-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>5.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample22</th>\n",
       "      <td>43.430629</td>\n",
       "      <td>05-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>5.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample23</th>\n",
       "      <td>64.544543</td>\n",
       "      <td>56-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>56.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample24</th>\n",
       "      <td>38.302485</td>\n",
       "      <td>10C-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>10.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample25</th>\n",
       "      <td>37.274027</td>\n",
       "      <td>10A-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>10.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample26</th>\n",
       "      <td>31.594701</td>\n",
       "      <td>08B-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>8.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample27</th>\n",
       "      <td>38.342321</td>\n",
       "      <td>10B-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>10.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample28</th>\n",
       "      <td>39.376480</td>\n",
       "      <td>07A-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>7.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample29</th>\n",
       "      <td>41.468048</td>\n",
       "      <td>90-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>90.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample30</th>\n",
       "      <td>65.373428</td>\n",
       "      <td>89-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>89.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample31</th>\n",
       "      <td>41.425684</td>\n",
       "      <td>90-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>90.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample32</th>\n",
       "      <td>62.312304</td>\n",
       "      <td>89-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>89.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample33</th>\n",
       "      <td>17.071664</td>\n",
       "      <td>03D-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>0.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample34</th>\n",
       "      <td>19.473595</td>\n",
       "      <td>03D-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>0.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample35</th>\n",
       "      <td>55.454335</td>\n",
       "      <td>96-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>96.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample36</th>\n",
       "      <td>81.034239</td>\n",
       "      <td>91-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>91.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample37</th>\n",
       "      <td>47.888099</td>\n",
       "      <td>92B-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>92.00</td>\n",
       "      <td>female</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample38</th>\n",
       "      <td>82.838960</td>\n",
       "      <td>91-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>91.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample39</th>\n",
       "      <td>17.769244</td>\n",
       "      <td>02-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>2.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample40</th>\n",
       "      <td>23.509896</td>\n",
       "      <td>02-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>2.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample41</th>\n",
       "      <td>41.505508</td>\n",
       "      <td>12-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>12.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample42</th>\n",
       "      <td>41.604927</td>\n",
       "      <td>12-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>12.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample43</th>\n",
       "      <td>41.846104</td>\n",
       "      <td>12B-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>12.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample44</th>\n",
       "      <td>32.252920</td>\n",
       "      <td>06-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>6.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample45</th>\n",
       "      <td>35.196626</td>\n",
       "      <td>12B-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>12.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample46</th>\n",
       "      <td>50.058842</td>\n",
       "      <td>19A-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>19.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample47</th>\n",
       "      <td>39.463079</td>\n",
       "      <td>19C-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>19.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample48</th>\n",
       "      <td>60.286239</td>\n",
       "      <td>68-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>68.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample49</th>\n",
       "      <td>65.275427</td>\n",
       "      <td>68-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>68.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample50</th>\n",
       "      <td>58.554913</td>\n",
       "      <td>94-M</td>\n",
       "      <td>MSN</td>\n",
       "      <td>94.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Lentiviral miR-9/9-124-CDM</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample51</th>\n",
       "      <td>56.334783</td>\n",
       "      <td>74-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>74.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample52</th>\n",
       "      <td>56.810323</td>\n",
       "      <td>94-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>94.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample53</th>\n",
       "      <td>55.448769</td>\n",
       "      <td>86-C</td>\n",
       "      <td>Fibroblasts</td>\n",
       "      <td>86.00</td>\n",
       "      <td>male</td>\n",
       "      <td>Untransformed</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            DNAmAge ExternalSampleID       Tissue    Age  Gender  \\\n",
       "SampleNo                                                           \n",
       "Sample1   43.430715            17B-C  Fibroblasts  17.00    male   \n",
       "Sample2   47.061787             72-C  Fibroblasts  72.00    male   \n",
       "Sample3   36.181183             01-M          MSN   1.00    male   \n",
       "Sample4   32.291262             01-C  Fibroblasts   1.00    male   \n",
       "Sample5   46.297813             44-M          MSN  44.00    male   \n",
       "Sample6   45.762984             44-C  Fibroblasts  44.00    male   \n",
       "Sample7   64.561971            92A-M          MSN  92.00    male   \n",
       "Sample8   71.634253            92A-C  Fibroblasts  92.00    male   \n",
       "Sample9   51.734553            92B-C  Fibroblasts  92.00  female   \n",
       "Sample10  45.524989             31-M          MSN  31.00    male   \n",
       "Sample11  40.024259             31-C  Fibroblasts  31.00    male   \n",
       "Sample12  36.185683          10B-C-2  Fibroblasts  10.00    male   \n",
       "Sample13  55.606625            11A-C  Fibroblasts  11.00    male   \n",
       "Sample14  31.185213            17A-C  Fibroblasts  17.00    male   \n",
       "Sample15  38.076881            05M-M          MSN   0.42    male   \n",
       "Sample16  26.929950            05M-C  Fibroblasts   0.42    male   \n",
       "Sample17  23.461219            07B-C  Fibroblasts   7.00    male   \n",
       "Sample18  37.497850            08A-C  Fibroblasts   8.00    male   \n",
       "Sample19  66.134065            92A-C  Fibroblasts  92.00    male   \n",
       "Sample20  40.591273             90-C  Fibroblasts  90.00    male   \n",
       "Sample21  21.565663             05-M          MSN   5.00    male   \n",
       "Sample22  43.430629             05-C  Fibroblasts   5.00    male   \n",
       "Sample23  64.544543             56-C  Fibroblasts  56.00    male   \n",
       "Sample24  38.302485            10C-C  Fibroblasts  10.00    male   \n",
       "Sample25  37.274027            10A-C  Fibroblasts  10.00    male   \n",
       "Sample26  31.594701            08B-C  Fibroblasts   8.00    male   \n",
       "Sample27  38.342321            10B-C  Fibroblasts  10.00    male   \n",
       "Sample28  39.376480            07A-C  Fibroblasts   7.00    male   \n",
       "Sample29  41.468048             90-M          MSN  90.00    male   \n",
       "Sample30  65.373428             89-M          MSN  89.00    male   \n",
       "Sample31  41.425684             90-C  Fibroblasts  90.00    male   \n",
       "Sample32  62.312304             89-C  Fibroblasts  89.00    male   \n",
       "Sample33  17.071664            03D-M          MSN   0.00    male   \n",
       "Sample34  19.473595            03D-C  Fibroblasts   0.00    male   \n",
       "Sample35  55.454335             96-M          MSN  96.00    male   \n",
       "Sample36  81.034239             91-M          MSN  91.00    male   \n",
       "Sample37  47.888099            92B-C  Fibroblasts  92.00  female   \n",
       "Sample38  82.838960             91-C  Fibroblasts  91.00    male   \n",
       "Sample39  17.769244             02-M          MSN   2.00    male   \n",
       "Sample40  23.509896             02-C  Fibroblasts   2.00    male   \n",
       "Sample41  41.505508             12-M          MSN  12.00    male   \n",
       "Sample42  41.604927             12-C  Fibroblasts  12.00    male   \n",
       "Sample43  41.846104            12B-M          MSN  12.00    male   \n",
       "Sample44  32.252920             06-C  Fibroblasts   6.00    male   \n",
       "Sample45  35.196626            12B-C  Fibroblasts  12.00    male   \n",
       "Sample46  50.058842            19A-C  Fibroblasts  19.00    male   \n",
       "Sample47  39.463079            19C-C  Fibroblasts  19.00    male   \n",
       "Sample48  60.286239             68-M          MSN  68.00    male   \n",
       "Sample49  65.275427             68-C  Fibroblasts  68.00    male   \n",
       "Sample50  58.554913             94-M          MSN  94.00    male   \n",
       "Sample51  56.334783             74-C  Fibroblasts  74.00    male   \n",
       "Sample52  56.810323             94-C  Fibroblasts  94.00    male   \n",
       "Sample53  55.448769             86-C  Fibroblasts  86.00    male   \n",
       "\n",
       "                TransformationStatus  MatchingPairHorvath  \n",
       "SampleNo                                                   \n",
       "Sample1                Untransformed                   -1  \n",
       "Sample2                Untransformed                   -1  \n",
       "Sample3   Lentiviral miR-9/9-124-CDM                    1  \n",
       "Sample4                Untransformed                    1  \n",
       "Sample5   Lentiviral miR-9/9-124-CDM                    9  \n",
       "Sample6                Untransformed                    9  \n",
       "Sample7   Lentiviral miR-9/9-124-CDM                   14  \n",
       "Sample8                Untransformed                   14  \n",
       "Sample9                Untransformed                   -1  \n",
       "Sample10  Lentiviral miR-9/9-124-CDM                    8  \n",
       "Sample11               Untransformed                    8  \n",
       "Sample12               Untransformed                   -1  \n",
       "Sample13               Untransformed                   -1  \n",
       "Sample14               Untransformed                   -1  \n",
       "Sample15  Lentiviral miR-9/9-124-CDM                    5  \n",
       "Sample16               Untransformed                    5  \n",
       "Sample17               Untransformed                   -1  \n",
       "Sample18               Untransformed                   -1  \n",
       "Sample19               Untransformed                   14  \n",
       "Sample20               Untransformed                   12  \n",
       "Sample21  Lentiviral miR-9/9-124-CDM                    4  \n",
       "Sample22               Untransformed                    4  \n",
       "Sample23               Untransformed                   -1  \n",
       "Sample24               Untransformed                   -1  \n",
       "Sample25               Untransformed                   -1  \n",
       "Sample26               Untransformed                   -1  \n",
       "Sample27               Untransformed                   -1  \n",
       "Sample28               Untransformed                   -1  \n",
       "Sample29  Lentiviral miR-9/9-124-CDM                   12  \n",
       "Sample30  Lentiviral miR-9/9-124-CDM                   11  \n",
       "Sample31               Untransformed                   12  \n",
       "Sample32               Untransformed                   11  \n",
       "Sample33  Lentiviral miR-9/9-124-CDM                    3  \n",
       "Sample34               Untransformed                    3  \n",
       "Sample35  Lentiviral miR-9/9-124-CDM                   -1  \n",
       "Sample36  Lentiviral miR-9/9-124-CDM                   13  \n",
       "Sample37               Untransformed                   -1  \n",
       "Sample38               Untransformed                   13  \n",
       "Sample39  Lentiviral miR-9/9-124-CDM                    2  \n",
       "Sample40               Untransformed                    2  \n",
       "Sample41  Lentiviral miR-9/9-124-CDM                    7  \n",
       "Sample42               Untransformed                    7  \n",
       "Sample43  Lentiviral miR-9/9-124-CDM                    6  \n",
       "Sample44               Untransformed                   -1  \n",
       "Sample45               Untransformed                    6  \n",
       "Sample46               Untransformed                   -1  \n",
       "Sample47               Untransformed                   -1  \n",
       "Sample48  Lentiviral miR-9/9-124-CDM                   10  \n",
       "Sample49               Untransformed                   10  \n",
       "Sample50  Lentiviral miR-9/9-124-CDM                   15  \n",
       "Sample51               Untransformed                   -1  \n",
       "Sample52               Untransformed                   15  \n",
       "Sample53               Untransformed                   -1  "
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_meta['MatchingPairHorvath'] = -1\n",
    "def try_dict(dic, x):\n",
    "    try:\n",
    "        return dic[x]\n",
    "    except:\n",
    "        return -1\n",
    "df_meta['MatchingPairHorvath'] = df_meta['ExternalSampleID'].apply(lambda x: try_dict(id2pair,x))\n",
    "# df_meta[df_meta.ExternalSampleID.isin(df_meta_paired.ExternalSampleIDFibroblast)].reset_index()\n",
    "# df_meta[df_meta.ExternalSampleID.isin(df_meta_paired.ExternalSampleIDMSN)].reset_index()\n",
    "# df_meta.loc[df_meta.index[df_meta.ExternalSampleID.isin(df_meta_paired.ExternalSampleIDFibroblast)], 'MatchingPairHorvath'] = df_meta_paired.MatchingPairHorvath.values\n",
    "df_meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "id": "fd85b649-991b-4930-b411-0e383c13b1d1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/tarkhov/Dropbox/Harvard/Science/Stochastic aging/data/elife18648/DNAmethylation/dat0NoobUpload.csv\n",
      "(485512, 53)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sample1</th>\n",
       "      <th>Sample2</th>\n",
       "      <th>Sample3</th>\n",
       "      <th>Sample4</th>\n",
       "      <th>Sample5</th>\n",
       "      <th>Sample6</th>\n",
       "      <th>Sample7</th>\n",
       "      <th>Sample8</th>\n",
       "      <th>Sample9</th>\n",
       "      <th>Sample10</th>\n",
       "      <th>...</th>\n",
       "      <th>Sample44</th>\n",
       "      <th>Sample45</th>\n",
       "      <th>Sample46</th>\n",
       "      <th>Sample47</th>\n",
       "      <th>Sample48</th>\n",
       "      <th>Sample49</th>\n",
       "      <th>Sample50</th>\n",
       "      <th>Sample51</th>\n",
       "      <th>Sample52</th>\n",
       "      <th>Sample53</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cg13869341</th>\n",
       "      <td>0.911558</td>\n",
       "      <td>0.865286</td>\n",
       "      <td>0.904650</td>\n",
       "      <td>0.896009</td>\n",
       "      <td>0.934046</td>\n",
       "      <td>0.930204</td>\n",
       "      <td>0.877897</td>\n",
       "      <td>0.891037</td>\n",
       "      <td>0.874961</td>\n",
       "      <td>0.900057</td>\n",
       "      <td>...</td>\n",
       "      <td>0.909558</td>\n",
       "      <td>0.950688</td>\n",
       "      <td>0.892038</td>\n",
       "      <td>0.926468</td>\n",
       "      <td>0.937595</td>\n",
       "      <td>0.972500</td>\n",
       "      <td>0.912934</td>\n",
       "      <td>0.968160</td>\n",
       "      <td>0.898411</td>\n",
       "      <td>0.889256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg14008030</th>\n",
       "      <td>0.571380</td>\n",
       "      <td>0.776371</td>\n",
       "      <td>0.724252</td>\n",
       "      <td>0.750212</td>\n",
       "      <td>0.691279</td>\n",
       "      <td>0.657718</td>\n",
       "      <td>0.633417</td>\n",
       "      <td>0.628771</td>\n",
       "      <td>0.580305</td>\n",
       "      <td>0.689459</td>\n",
       "      <td>...</td>\n",
       "      <td>0.573300</td>\n",
       "      <td>0.691607</td>\n",
       "      <td>0.600626</td>\n",
       "      <td>0.633259</td>\n",
       "      <td>0.809920</td>\n",
       "      <td>0.870201</td>\n",
       "      <td>0.515003</td>\n",
       "      <td>0.696266</td>\n",
       "      <td>0.504167</td>\n",
       "      <td>0.640626</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg12045430</th>\n",
       "      <td>0.058212</td>\n",
       "      <td>0.058891</td>\n",
       "      <td>0.061507</td>\n",
       "      <td>0.058667</td>\n",
       "      <td>0.048445</td>\n",
       "      <td>0.041869</td>\n",
       "      <td>0.077344</td>\n",
       "      <td>0.050087</td>\n",
       "      <td>0.051238</td>\n",
       "      <td>0.064071</td>\n",
       "      <td>...</td>\n",
       "      <td>0.061662</td>\n",
       "      <td>0.070248</td>\n",
       "      <td>0.054186</td>\n",
       "      <td>0.064938</td>\n",
       "      <td>0.062726</td>\n",
       "      <td>0.053776</td>\n",
       "      <td>0.075051</td>\n",
       "      <td>0.057004</td>\n",
       "      <td>0.061515</td>\n",
       "      <td>0.076911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg20826792</th>\n",
       "      <td>0.151400</td>\n",
       "      <td>0.162204</td>\n",
       "      <td>0.118754</td>\n",
       "      <td>0.118632</td>\n",
       "      <td>0.119289</td>\n",
       "      <td>0.076153</td>\n",
       "      <td>0.161194</td>\n",
       "      <td>0.196185</td>\n",
       "      <td>0.108328</td>\n",
       "      <td>0.098255</td>\n",
       "      <td>...</td>\n",
       "      <td>0.071205</td>\n",
       "      <td>0.098902</td>\n",
       "      <td>0.108929</td>\n",
       "      <td>0.142121</td>\n",
       "      <td>0.186734</td>\n",
       "      <td>0.110953</td>\n",
       "      <td>0.176312</td>\n",
       "      <td>0.129514</td>\n",
       "      <td>0.175018</td>\n",
       "      <td>0.101644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00381604</th>\n",
       "      <td>0.042924</td>\n",
       "      <td>0.049899</td>\n",
       "      <td>0.034948</td>\n",
       "      <td>0.043083</td>\n",
       "      <td>0.026124</td>\n",
       "      <td>0.026349</td>\n",
       "      <td>0.039345</td>\n",
       "      <td>0.037826</td>\n",
       "      <td>0.036229</td>\n",
       "      <td>0.038190</td>\n",
       "      <td>...</td>\n",
       "      <td>0.035168</td>\n",
       "      <td>0.036718</td>\n",
       "      <td>0.028564</td>\n",
       "      <td>0.040430</td>\n",
       "      <td>0.038562</td>\n",
       "      <td>0.034046</td>\n",
       "      <td>0.043510</td>\n",
       "      <td>0.045250</td>\n",
       "      <td>0.038708</td>\n",
       "      <td>0.035456</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 53 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Sample1   Sample2   Sample3   Sample4   Sample5   Sample6  \\\n",
       "ID                                                                       \n",
       "cg13869341  0.911558  0.865286  0.904650  0.896009  0.934046  0.930204   \n",
       "cg14008030  0.571380  0.776371  0.724252  0.750212  0.691279  0.657718   \n",
       "cg12045430  0.058212  0.058891  0.061507  0.058667  0.048445  0.041869   \n",
       "cg20826792  0.151400  0.162204  0.118754  0.118632  0.119289  0.076153   \n",
       "cg00381604  0.042924  0.049899  0.034948  0.043083  0.026124  0.026349   \n",
       "\n",
       "             Sample7   Sample8   Sample9  Sample10  ...  Sample44  Sample45  \\\n",
       "ID                                                  ...                       \n",
       "cg13869341  0.877897  0.891037  0.874961  0.900057  ...  0.909558  0.950688   \n",
       "cg14008030  0.633417  0.628771  0.580305  0.689459  ...  0.573300  0.691607   \n",
       "cg12045430  0.077344  0.050087  0.051238  0.064071  ...  0.061662  0.070248   \n",
       "cg20826792  0.161194  0.196185  0.108328  0.098255  ...  0.071205  0.098902   \n",
       "cg00381604  0.039345  0.037826  0.036229  0.038190  ...  0.035168  0.036718   \n",
       "\n",
       "            Sample46  Sample47  Sample48  Sample49  Sample50  Sample51  \\\n",
       "ID                                                                       \n",
       "cg13869341  0.892038  0.926468  0.937595  0.972500  0.912934  0.968160   \n",
       "cg14008030  0.600626  0.633259  0.809920  0.870201  0.515003  0.696266   \n",
       "cg12045430  0.054186  0.064938  0.062726  0.053776  0.075051  0.057004   \n",
       "cg20826792  0.108929  0.142121  0.186734  0.110953  0.176312  0.129514   \n",
       "cg00381604  0.028564  0.040430  0.038562  0.034046  0.043510  0.045250   \n",
       "\n",
       "            Sample52  Sample53  \n",
       "ID                              \n",
       "cg13869341  0.898411  0.889256  \n",
       "cg14008030  0.504167  0.640626  \n",
       "cg12045430  0.061515  0.076911  \n",
       "cg20826792  0.175018  0.101644  \n",
       "cg00381604  0.038708  0.035456  \n",
       "\n",
       "[5 rows x 53 columns]"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_path = Path.joinpath(cwd, project_dir, \"DNAmethylation/dat0NoobUpload.csv\")\n",
    "print(data_path)\n",
    "cols = pd.read_csv(data_path, nrows=0).columns\n",
    "# df_percent = pd.read_csv(data_path, index_col=0, usecols=[cols[0]] + cols[1::2].tolist())\n",
    "df_percent = pd.read_csv(data_path, index_col=0)\n",
    "# df_percent.columns = [x.split('_')[0] for x in df_percent.columns]\n",
    "# df_percent = mtools.get_methylation_df(data_path, typ='percents')\n",
    "# df_percent.columns = df_percent.columns.get_level_values(1)\n",
    "df_percent.columns = [''.join(x.split('No')) for x in df_percent.columns]\n",
    "print(df_percent.shape)\n",
    "df_percent.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "983cf48e-6099-4da7-b77a-f4470d674153",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "5bace01b-7243-4c09-ac99-1c63d03f9182",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_tmp = pd.read_csv(Path.joinpath(cwd, project_dir, \"Datasets_Transcriptome_And_microRNAome/Processed_data/Agilent_Human4X44K_Expression_q0.95.table.txt\"), sep = '\\t')\n",
    "# df_tmp = pd.read_csv(Path.joinpath(cwd, project_dir, \"Datasets_Transcriptome_And_microRNAome/Processed_data/Agilent_Human4X44K_Merged_Expression_q0.95.table.txt\"), sep = '\\t')\n",
    "# df_tmp = pd.read_csv(Path.joinpath(cwd, project_dir, \"Datasets_Transcriptome_And_microRNAome/Processed_data/Expression_miRNA_human.txt\"), sep = '\\t')\n",
    "# df_tmp = pd.read_csv(Path.joinpath(cwd, project_dir, \"Datasets_Transcriptome_And_microRNAome/DEGs/miRNA/DEG_Fibroblast_Young_Vs_Old.table.txt\"), sep = '\\t')\n",
    "# df_tmp = pd.read_csv(Path.joinpath(cwd, project_dir, \"Datasets_Transcriptome_And_microRNAome/DEGs/miRNA/DEG_MSN_Young_Vs_Old.table.txt\"), sep = '\\t')\n",
    "# df_tmp = pd.read_csv(Path.joinpath(cwd, project_dir, \"Datasets_Transcriptome_And_microRNAome/DEGs/miRNA/DEG_Old_Fibroblast_Vs_MSN.table.txt\"), sep = '\\t')\n",
    "# df_tmp = pd.read_csv(Path.joinpath(cwd, project_dir, \"Datasets_Transcriptome_And_microRNAome/DEGs/miRNA/DEG_Young_Fibroblast_Vs_MSN.table.txt\"), sep = '\\t')\n",
    "# df_tmp = pd.read_csv(Path.joinpath(cwd, project_dir, \"Datasets_Transcriptome_And_microRNAome/DEGs/mRNA/DEG_Fibroblast_Young_Vs_Old.table.txt\"), sep = '\\t')\n",
    "# df_tmp = pd.read_csv(Path.joinpath(cwd, project_dir, \"Datasets_Transcriptome_And_microRNAome/DEGs/mRNA/DEG_MSN_Young_Vs_Old.table.txt\"), sep = '\\t')\n",
    "# df_tmp = pd.read_csv(Path.joinpath(cwd, project_dir, \"Datasets_Transcriptome_And_microRNAome/DEGs/mRNA/DEG_Old_Fibroblast_Vs_MSN_Old.table.txt\"), sep = '\\t')\n",
    "# df_tmp = pd.read_csv(Path.joinpath(cwd, project_dir, \"Datasets_Transcriptome_And_microRNAome/DEGs/mRNA/DEG_Young_Fibroblast_Vs_MSN_Old.table.txt\"), sep = '\\t')\n",
    "\n",
    "# print(df_tmp.shape)\n",
    "# df_tmp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "id": "1afc71bb-bc83-4bf8-a40a-70e5ec0c731c",
   "metadata": {},
   "outputs": [],
   "source": [
    "ad = sc.AnnData(df_percent.loc[:,df_meta.index])\n",
    "ad.var = pd.concat((ad.var, df_meta.loc[ad.var.index,:]), axis=1)\n",
    "ad.X = ad.X.astype(np.float16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "id": "83fd1419-0194-47fa-9106-4d4a2b2d39fc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'ExternalSampleID' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Tissue' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Gender' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'TransformationStatus' as categorical\n"
     ]
    }
   ],
   "source": [
    "ad.write_h5ad(Path.joinpath(project_dir, \"{}.h5ad\".format(project_id)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7289bd3-2d66-4706-9cd7-3b55c0decf32",
   "metadata": {},
   "source": [
    "# iPSC experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "539d4cf3-12be-45ea-9089-c8c8236f4b80",
   "metadata": {},
   "outputs": [],
   "source": [
    "cwd = Path.cwd()\n",
    "project_dir = Path.joinpath(cwd, Path('data/N77.2020-9165IPSmammalGladyshevSangGooLee'))\n",
    "project_id = 'N77.2020-9165IPSmammalGladyshevSangGooLee'\n",
    "project_samples_dir = 'GSE179039_RAW'\n",
    "metainfo_path = '41467_2022_27959_MOESM8_ESM.xlsx'\n",
    "metainfo_new_path = '41467_2022_27959_MOESM8_ESM_proc.csv'\n",
    "gl_metainfo_path = Path.joinpath(project_dir, metainfo_path)\n",
    "gl_metainfo_new_path = Path.joinpath(project_dir, metainfo_new_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "69071898-3a11-4c65-a1cb-a7c78491319b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/tarkhov/Dropbox/Harvard/Science/Stochastic aging/data/N77.2020-9165IPSmammalGladyshevSangGooLee/N77.2020-9165IPSmammalGladyshevSangGooLee.csv\n"
     ]
    },
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: '/Users/tarkhov/Dropbox/Harvard/Science/Stochastic aging/data/N77.2020-9165IPSmammalGladyshevSangGooLee/N77.2020-9165IPSmammalGladyshevSangGooLee.csv'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [3]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28mprint\u001b[39m(data_path)\n\u001b[1;32m      4\u001b[0m \u001b[38;5;66;03m# df_coverage = mtools.get_methylation_df(data_path, typ='coverage')\u001b[39;00m\n\u001b[0;32m----> 5\u001b[0m df_percent \u001b[38;5;241m=\u001b[39m \u001b[43mmtools\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_methylation_df\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtyp\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mpercents\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m      6\u001b[0m \u001b[38;5;28mprint\u001b[39m(df_percent\u001b[38;5;241m.\u001b[39mshape)\n\u001b[1;32m      7\u001b[0m df_percent\u001b[38;5;241m.\u001b[39mhead()\n",
      "File \u001b[0;32m~/Dropbox/Harvard/Science/Stochastic aging/meth/tools.py:68\u001b[0m, in \u001b[0;36mget_methylation_df\u001b[0;34m(data_path, typ)\u001b[0m\n\u001b[1;32m     67\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_methylation_df\u001b[39m(data_path, typ\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpercents\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[0;32m---> 68\u001b[0m     cols \u001b[38;5;241m=\u001b[39m \u001b[43mpd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_csv\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msep\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;130;43;01m\\t\u001b[39;49;00m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnrows\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mcolumns\n\u001b[1;32m     69\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m typ \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcoverage\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[1;32m     70\u001b[0m         df \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mread_csv(data_path, sep\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\t\u001b[39;00m\u001b[38;5;124m'\u001b[39m, index_col\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m, usecols\u001b[38;5;241m=\u001b[39mcols[::\u001b[38;5;241m2\u001b[39m])\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/py310/lib/python3.10/site-packages/pandas/util/_decorators.py:311\u001b[0m, in \u001b[0;36mdeprecate_nonkeyword_arguments.<locals>.decorate.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    305\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(args) \u001b[38;5;241m>\u001b[39m num_allow_args:\n\u001b[1;32m    306\u001b[0m     warnings\u001b[38;5;241m.\u001b[39mwarn(\n\u001b[1;32m    307\u001b[0m         msg\u001b[38;5;241m.\u001b[39mformat(arguments\u001b[38;5;241m=\u001b[39marguments),\n\u001b[1;32m    308\u001b[0m         \u001b[38;5;167;01mFutureWarning\u001b[39;00m,\n\u001b[1;32m    309\u001b[0m         stacklevel\u001b[38;5;241m=\u001b[39mstacklevel,\n\u001b[1;32m    310\u001b[0m     )\n\u001b[0;32m--> 311\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/py310/lib/python3.10/site-packages/pandas/io/parsers/readers.py:586\u001b[0m, in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, error_bad_lines, warn_bad_lines, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options)\u001b[0m\n\u001b[1;32m    571\u001b[0m kwds_defaults \u001b[38;5;241m=\u001b[39m _refine_defaults_read(\n\u001b[1;32m    572\u001b[0m     dialect,\n\u001b[1;32m    573\u001b[0m     delimiter,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    582\u001b[0m     defaults\u001b[38;5;241m=\u001b[39m{\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdelimiter\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m\"\u001b[39m},\n\u001b[1;32m    583\u001b[0m )\n\u001b[1;32m    584\u001b[0m kwds\u001b[38;5;241m.\u001b[39mupdate(kwds_defaults)\n\u001b[0;32m--> 586\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_read\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfilepath_or_buffer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/py310/lib/python3.10/site-packages/pandas/io/parsers/readers.py:482\u001b[0m, in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    479\u001b[0m _validate_names(kwds\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnames\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m))\n\u001b[1;32m    481\u001b[0m \u001b[38;5;66;03m# Create the parser.\u001b[39;00m\n\u001b[0;32m--> 482\u001b[0m parser \u001b[38;5;241m=\u001b[39m \u001b[43mTextFileReader\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfilepath_or_buffer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    484\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m chunksize \u001b[38;5;129;01mor\u001b[39;00m iterator:\n\u001b[1;32m    485\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m parser\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/py310/lib/python3.10/site-packages/pandas/io/parsers/readers.py:811\u001b[0m, in \u001b[0;36mTextFileReader.__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m    808\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhas_index_names\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m kwds:\n\u001b[1;32m    809\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moptions[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhas_index_names\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m kwds[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhas_index_names\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[0;32m--> 811\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_engine \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_engine\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mengine\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/py310/lib/python3.10/site-packages/pandas/io/parsers/readers.py:1040\u001b[0m, in \u001b[0;36mTextFileReader._make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m   1036\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m   1037\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUnknown engine: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mengine\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m (valid options are \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmapping\u001b[38;5;241m.\u001b[39mkeys()\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m)\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   1038\u001b[0m     )\n\u001b[1;32m   1039\u001b[0m \u001b[38;5;66;03m# error: Too many arguments for \"ParserBase\"\u001b[39;00m\n\u001b[0;32m-> 1040\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mmapping\u001b[49m\u001b[43m[\u001b[49m\u001b[43mengine\u001b[49m\u001b[43m]\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mf\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/py310/lib/python3.10/site-packages/pandas/io/parsers/c_parser_wrapper.py:51\u001b[0m, in \u001b[0;36mCParserWrapper.__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m     48\u001b[0m kwds[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124musecols\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39musecols\n\u001b[1;32m     50\u001b[0m \u001b[38;5;66;03m# open handles\u001b[39;00m\n\u001b[0;32m---> 51\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_open_handles\u001b[49m\u001b[43m(\u001b[49m\u001b[43msrc\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     52\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhandles \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m     54\u001b[0m \u001b[38;5;66;03m# Have to pass int, would break tests using TextReader directly otherwise :(\u001b[39;00m\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/py310/lib/python3.10/site-packages/pandas/io/parsers/base_parser.py:222\u001b[0m, in \u001b[0;36mParserBase._open_handles\u001b[0;34m(self, src, kwds)\u001b[0m\n\u001b[1;32m    218\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_open_handles\u001b[39m(\u001b[38;5;28mself\u001b[39m, src: FilePathOrBuffer, kwds: \u001b[38;5;28mdict\u001b[39m[\u001b[38;5;28mstr\u001b[39m, Any]) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    219\u001b[0m     \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    220\u001b[0m \u001b[38;5;124;03m    Let the readers open IOHandles after they are done with their potential raises.\u001b[39;00m\n\u001b[1;32m    221\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 222\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhandles \u001b[38;5;241m=\u001b[39m \u001b[43mget_handle\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    223\u001b[0m \u001b[43m        \u001b[49m\u001b[43msrc\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    224\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mr\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m    225\u001b[0m \u001b[43m        \u001b[49m\u001b[43mencoding\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mencoding\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    226\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcompression\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcompression\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    227\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmemory_map\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mmemory_map\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    228\u001b[0m \u001b[43m        \u001b[49m\u001b[43mstorage_options\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mstorage_options\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    229\u001b[0m \u001b[43m        \u001b[49m\u001b[43merrors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mencoding_errors\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mstrict\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    230\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/py310/lib/python3.10/site-packages/pandas/io/common.py:702\u001b[0m, in \u001b[0;36mget_handle\u001b[0;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[1;32m    697\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(handle, \u001b[38;5;28mstr\u001b[39m):\n\u001b[1;32m    698\u001b[0m     \u001b[38;5;66;03m# Check whether the filename is to be opened in binary mode.\u001b[39;00m\n\u001b[1;32m    699\u001b[0m     \u001b[38;5;66;03m# Binary mode does not support 'encoding' and 'newline'.\u001b[39;00m\n\u001b[1;32m    700\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m ioargs\u001b[38;5;241m.\u001b[39mencoding \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m ioargs\u001b[38;5;241m.\u001b[39mmode:\n\u001b[1;32m    701\u001b[0m         \u001b[38;5;66;03m# Encoding\u001b[39;00m\n\u001b[0;32m--> 702\u001b[0m         handle \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mopen\u001b[39;49m\u001b[43m(\u001b[49m\n\u001b[1;32m    703\u001b[0m \u001b[43m            \u001b[49m\u001b[43mhandle\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    704\u001b[0m \u001b[43m            \u001b[49m\u001b[43mioargs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    705\u001b[0m \u001b[43m            \u001b[49m\u001b[43mencoding\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mioargs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencoding\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    706\u001b[0m \u001b[43m            \u001b[49m\u001b[43merrors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43merrors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    707\u001b[0m \u001b[43m            \u001b[49m\u001b[43mnewline\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m    708\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    709\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    710\u001b[0m         \u001b[38;5;66;03m# Binary mode\u001b[39;00m\n\u001b[1;32m    711\u001b[0m         handle \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mopen\u001b[39m(handle, ioargs\u001b[38;5;241m.\u001b[39mmode)\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/Users/tarkhov/Dropbox/Harvard/Science/Stochastic aging/data/N77.2020-9165IPSmammalGladyshevSangGooLee/N77.2020-9165IPSmammalGladyshevSangGooLee.csv'"
     ]
    }
   ],
   "source": [
    "data_path = Path.joinpath(cwd, project_dir, \"{}.csv\".format(project_id))\n",
    "print(data_path)\n",
    "\n",
    "# df_coverage = mtools.get_methylation_df(data_path, typ='coverage')\n",
    "df_percent = mtools.get_methylation_df(data_path, typ='percents')\n",
    "print(df_percent.shape)\n",
    "df_percent.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "42c0d512-b1e7-48f4-9284-62be551727a0",
   "metadata": {},
   "source": [
    "# NMR methylation data — GSE179039 (Csaba)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "41f0feb4-fc3c-4b45-b9ee-5bb501b1900d",
   "metadata": {},
   "outputs": [],
   "source": [
    "cwd = Path.cwd()\n",
    "project_dir = Path.joinpath(cwd, Path('data/GSE179039'))\n",
    "project_id = 'GSE179039'\n",
    "project_samples_dir = 'GSE179039_RAW'\n",
    "metainfo_path = '41467_2022_27959_MOESM8_ESM.xlsx'\n",
    "metainfo_new_path = '41467_2022_27959_MOESM8_ESM_proc.csv'\n",
    "gl_metainfo_path = Path.joinpath(project_dir, metainfo_path)\n",
    "gl_metainfo_new_path = Path.joinpath(project_dir, metainfo_new_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "2467fdff-ecc2-4645-96d0-de85fa632ebe",
   "metadata": {},
   "outputs": [],
   "source": [
    "path_samples = Path.joinpath(project_dir, project_samples_dir)\n",
    "gl_path_samples = Path.joinpath(cwd, path_samples)\n",
    "all_samples = os.listdir(gl_path_samples)\n",
    "id_to_gsm = dict(zip([x.split('_')[1] for x in all_samples], [x.split('_')[0] for x in all_samples]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "e9edea83-79ab-4658-ae76-9a2f45efd0cb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4978505, 107)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GSM5404291</th>\n",
       "      <th>GSM5404264</th>\n",
       "      <th>GSM5404283</th>\n",
       "      <th>GSM5404302</th>\n",
       "      <th>GSM5404312</th>\n",
       "      <th>GSM5404254</th>\n",
       "      <th>GSM5404269</th>\n",
       "      <th>GSM5404267</th>\n",
       "      <th>GSM5404250</th>\n",
       "      <th>GSM5404295</th>\n",
       "      <th>...</th>\n",
       "      <th>GSM5404289</th>\n",
       "      <th>GSM5404303</th>\n",
       "      <th>GSM5404305</th>\n",
       "      <th>GSM5404271</th>\n",
       "      <th>GSM5404247</th>\n",
       "      <th>GSM5404336</th>\n",
       "      <th>GSM5404328</th>\n",
       "      <th>GSM5404260</th>\n",
       "      <th>GSM5404331</th>\n",
       "      <th>GSM5404321</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>JH602090.1_70815</th>\n",
       "      <td>8.333333</td>\n",
       "      <td>10.526316</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11.111111</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JH602090.1_70846</th>\n",
       "      <td>5.555556</td>\n",
       "      <td>2.631579</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.174603</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JH602090.1_70860</th>\n",
       "      <td>8.333333</td>\n",
       "      <td>11.111111</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.836066</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JH602090.1_97508</th>\n",
       "      <td>93.333333</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>86.666667</td>\n",
       "      <td>NaN</td>\n",
       "      <td>94.285714</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>87.096774</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>93.939394</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>93.333333</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JH602090.1_97531</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>97.142857</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>93.750000</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>93.548387</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 107 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  GSM5404291  GSM5404264  GSM5404283  GSM5404302  GSM5404312  \\\n",
       "index                                                                          \n",
       "JH602090.1_70815    8.333333   10.526316         NaN         NaN         NaN   \n",
       "JH602090.1_70846    5.555556    2.631579         NaN         NaN         NaN   \n",
       "JH602090.1_70860    8.333333   11.111111         NaN         NaN         NaN   \n",
       "JH602090.1_97508   93.333333         NaN         NaN   86.666667         NaN   \n",
       "JH602090.1_97531  100.000000         NaN         NaN  100.000000         NaN   \n",
       "\n",
       "                  GSM5404254  GSM5404269  GSM5404267  GSM5404250  GSM5404295  \\\n",
       "index                                                                          \n",
       "JH602090.1_70815         NaN         NaN         NaN         NaN         NaN   \n",
       "JH602090.1_70846         NaN         NaN         NaN         NaN         NaN   \n",
       "JH602090.1_70860         NaN         NaN         NaN         NaN         NaN   \n",
       "JH602090.1_97508   94.285714         NaN         NaN         NaN   87.096774   \n",
       "JH602090.1_97531   97.142857         NaN         NaN         NaN   93.750000   \n",
       "\n",
       "                  ...  GSM5404289  GSM5404303  GSM5404305  GSM5404271  \\\n",
       "index             ...                                                   \n",
       "JH602090.1_70815  ...         NaN         NaN         NaN         NaN   \n",
       "JH602090.1_70846  ...         NaN         NaN         NaN         NaN   \n",
       "JH602090.1_70860  ...         NaN         NaN         NaN         NaN   \n",
       "JH602090.1_97508  ...         NaN         NaN   93.939394         NaN   \n",
       "JH602090.1_97531  ...         NaN         NaN  100.000000         NaN   \n",
       "\n",
       "                  GSM5404247  GSM5404336  GSM5404328  GSM5404260  GSM5404331  \\\n",
       "index                                                                          \n",
       "JH602090.1_70815         NaN         NaN         NaN   11.111111         NaN   \n",
       "JH602090.1_70846         NaN         NaN         NaN    3.174603         NaN   \n",
       "JH602090.1_70860         NaN         NaN         NaN    9.836066         NaN   \n",
       "JH602090.1_97508         NaN   93.333333         NaN         NaN         NaN   \n",
       "JH602090.1_97531         NaN   93.548387         NaN         NaN         NaN   \n",
       "\n",
       "                  GSM5404321  \n",
       "index                         \n",
       "JH602090.1_70815         NaN  \n",
       "JH602090.1_70846         NaN  \n",
       "JH602090.1_70860         NaN  \n",
       "JH602090.1_97508         NaN  \n",
       "JH602090.1_97531         NaN  \n",
       "\n",
       "[5 rows x 107 columns]"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_path = Path.joinpath(cwd, project_dir, \"{}.csv\".format(project_id))\n",
    "\n",
    "df_percent = mtools.get_methylation_df(data_path, typ='percents')\n",
    "df_coverage = mtools.get_methylation_df(data_path, typ='coverage')\n",
    "print(df_percent.shape)\n",
    "df_percent.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "c1dc5aaa-948f-4393-b4c8-bcbd6008cb08",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 107 entries, GSM5404242 to GSM5404348\n",
      "Data columns (total 4 columns):\n",
      " #   Column           Non-Null Count  Dtype   \n",
      "---  ------           --------------  -----   \n",
      " 0   Sample Name      107 non-null    category\n",
      " 1   Sex              88 non-null     category\n",
      " 2   Breeding status  107 non-null    category\n",
      " 3   Age              107 non-null    float64 \n",
      "dtypes: category(3), float64(1)\n",
      "memory usage: 7.1+ KB\n"
     ]
    }
   ],
   "source": [
    "df_meta = pd.read_excel(gl_metainfo_path, index_col=0)\n",
    "df_meta['GSM'] = df_meta.reset_index()['Sample Name'].apply(lambda x: id_to_gsm[x]).values\n",
    "df_meta.reset_index(inplace=True)\n",
    "df_meta.set_index('GSM', inplace=True)\n",
    "df_meta['Age'] = df_meta['Age (years)']\n",
    "to_category = ['Sex', 'Breeding status', 'Sample Name']\n",
    "df_meta[to_category] = df_meta[to_category].astype(\"category\")\n",
    "df_meta.drop(\"Age (years)\", axis=1, inplace=True)\n",
    "df_meta.to_csv(gl_metainfo_new_path)\n",
    "# df_meta.head()\n",
    "df_meta.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "0c088e03-8973-4719-ac31-1e8398d40007",
   "metadata": {},
   "outputs": [],
   "source": [
    "ad = sc.AnnData(df_percent)\n",
    "ad.var = pd.concat((ad.var, df_meta.loc[ad.var.index,:]), axis=1)\n",
    "ad.X = ad.X.astype(np.float16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "d850228f-6f50-4022-9c57-dcfbd9a115fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "ad.write_h5ad(Path.joinpath(project_dir, \"{}.h5ad\".format(project_id)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba6aed1c-ae4a-4302-8b3f-3c71df801205",
   "metadata": {},
   "source": [
    "# Mice methylation data — GSE80672 (Petkovich)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "09660858-44dc-4da3-9c9c-2d98d7a5ff5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "cwd = Path.cwd()\n",
    "project_dir = Path.joinpath(cwd, Path('data/GSE80672'))\n",
    "project_id = 'GSE80672'\n",
    "project_samples_dir = 'GSE80672_RAW'\n",
    "\n",
    "soft_path = Path.joinpath(cwd, project_dir, '{}_family.soft.gz'.format(project_id))\n",
    "metainfo_path = '1-s2.0-S1550413117301687-mmc2.xlsx'\n",
    "gl_metainfo_path = Path.joinpath(project_dir, metainfo_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a901f761-1e83-4cd2-aaee-020adc769580",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# gse = GEOparse.get_GEO(filepath=str(soft_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9107f5ec-9e09-4e8c-b4df-de07937e8b27",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.67 1.17 10 18 23 27 20 21 6 5 '2.5 (at isolation)' 3 4 8 12 16 22 24 26\n",
      " 28 30 32 34 35]\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 255 entries, 20D02 to MIPSL3\n",
      "Data columns (total 5 columns):\n",
      " #   Column                    Non-Null Count  Dtype   \n",
      "---  ------                    --------------  -----   \n",
      " 0   Strain/Condition          255 non-null    category\n",
      " 1   Age                       255 non-null    float16 \n",
      " 2   Sex                       255 non-null    category\n",
      " 3   Prepared for Sequencing   255 non-null    category\n",
      " 4   Used in Subset 1/2        141 non-null    category\n",
      "dtypes: category(4), float16(1)\n",
      "memory usage: 4.5+ KB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/v_/gtsrj3xs0_131ltxwg51r_f40000gn/T/ipykernel_9964/2191250736.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_meta[\"Age\"][df_meta[\"Age\"] == '2.5 (at isolation)'] = 2.5\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Strain/Condition</th>\n",
       "      <th>Age</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Prepared for Sequencing</th>\n",
       "      <th>Used in Subset 1/2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sample Name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20D02</th>\n",
       "      <td>C57BL/6_NIA/BWH</td>\n",
       "      <td>0.669922</td>\n",
       "      <td>Male</td>\n",
       "      <td>D.A. Petkovich</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20D03</th>\n",
       "      <td>C57BL/6_NIA/BWH</td>\n",
       "      <td>0.669922</td>\n",
       "      <td>Male</td>\n",
       "      <td>D.A. Petkovich</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20D04</th>\n",
       "      <td>C57BL/6_NIA/BWH</td>\n",
       "      <td>0.669922</td>\n",
       "      <td>Male</td>\n",
       "      <td>D.A. Petkovich</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20D05</th>\n",
       "      <td>C57BL/6_NIA/BWH</td>\n",
       "      <td>0.669922</td>\n",
       "      <td>Male</td>\n",
       "      <td>D.A. Petkovich</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20D06</th>\n",
       "      <td>C57BL/6_NIA/BWH</td>\n",
       "      <td>0.669922</td>\n",
       "      <td>Male</td>\n",
       "      <td>D.A. Petkovich</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Strain/Condition       Age   Sex Prepared for Sequencing   \\\n",
       "Sample Name                                                             \n",
       "20D02        C57BL/6_NIA/BWH  0.669922  Male           D.A. Petkovich   \n",
       "20D03        C57BL/6_NIA/BWH  0.669922  Male           D.A. Petkovich   \n",
       "20D04        C57BL/6_NIA/BWH  0.669922  Male           D.A. Petkovich   \n",
       "20D05        C57BL/6_NIA/BWH  0.669922  Male           D.A. Petkovich   \n",
       "20D06        C57BL/6_NIA/BWH  0.669922  Male           D.A. Petkovich   \n",
       "\n",
       "            Used in Subset 1/2  \n",
       "Sample Name                     \n",
       "20D02                      NaN  \n",
       "20D03                      NaN  \n",
       "20D04                      NaN  \n",
       "20D05                      NaN  \n",
       "20D06                      NaN  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_meta = pd.read_excel(gl_metainfo_path, skiprows=3, index_col=0)\n",
    "to_category = ['Sex', 'Strain/Condition', 'Used in Subset 1/2', 'Prepared for Sequencing ']\n",
    "df_meta.reset_index(inplace=True)\n",
    "df_meta['Sample Name'] = df_meta['Sample Name'].apply(str.upper)\n",
    "df_meta.set_index('Sample Name', inplace=True)\n",
    "df_meta[to_category] = df_meta[to_category].astype(\"category\")\n",
    "print(df_meta[\"Age\"].unique())\n",
    "df_meta[\"Age\"][df_meta[\"Age\"] == '2.5 (at isolation)'] = 2.5\n",
    "df_meta[\"Age\"] = df_meta[\"Age\"].astype(np.float16)\n",
    "df_meta.drop([\"Aligned PE Reads\",], axis=1, inplace=True)\n",
    "df_meta.info()\n",
    "df_meta.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f74a84a4-4389-4e8c-896b-788093eab007",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/tarkhov/Dropbox/Harvard/Science/Stochastic aging/data/GSE80672/GSE80672.csv\n",
      "(1976056, 255)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>M3204</th>\n",
       "      <th>M3503R</th>\n",
       "      <th>GX2467</th>\n",
       "      <th>DW33</th>\n",
       "      <th>M2402</th>\n",
       "      <th>M0806</th>\n",
       "      <th>M3405</th>\n",
       "      <th>B6D2F1_CR2705</th>\n",
       "      <th>GX2470</th>\n",
       "      <th>B6_CR2302</th>\n",
       "      <th>...</th>\n",
       "      <th>M0404</th>\n",
       "      <th>M2202</th>\n",
       "      <th>M2804</th>\n",
       "      <th>DW1</th>\n",
       "      <th>GX2519</th>\n",
       "      <th>B6D2F1_CR2101</th>\n",
       "      <th>GX2481</th>\n",
       "      <th>GX2596</th>\n",
       "      <th>M0606</th>\n",
       "      <th>35D02</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>chrY_10165553</th>\n",
       "      <td>0.0</td>\n",
       "      <td>6.25</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>14.285714</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>10.526316</td>\n",
       "      <td>6.25</td>\n",
       "      <td>4.347826</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chrY_10165561</th>\n",
       "      <td>92.0</td>\n",
       "      <td>81.25</td>\n",
       "      <td>NaN</td>\n",
       "      <td>83.333333</td>\n",
       "      <td>92.307692</td>\n",
       "      <td>87.50</td>\n",
       "      <td>83.333333</td>\n",
       "      <td>63.636364</td>\n",
       "      <td>85.714286</td>\n",
       "      <td>90.909091</td>\n",
       "      <td>...</td>\n",
       "      <td>89.473684</td>\n",
       "      <td>87.50</td>\n",
       "      <td>95.652174</td>\n",
       "      <td>73.333333</td>\n",
       "      <td>80.0</td>\n",
       "      <td>64.285714</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.0</td>\n",
       "      <td>77.777778</td>\n",
       "      <td>70.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chrY_10165579</th>\n",
       "      <td>80.0</td>\n",
       "      <td>68.75</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33.333333</td>\n",
       "      <td>76.923077</td>\n",
       "      <td>93.75</td>\n",
       "      <td>88.888889</td>\n",
       "      <td>36.363636</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>90.909091</td>\n",
       "      <td>...</td>\n",
       "      <td>94.736842</td>\n",
       "      <td>68.75</td>\n",
       "      <td>86.956522</td>\n",
       "      <td>93.333333</td>\n",
       "      <td>60.0</td>\n",
       "      <td>64.285714</td>\n",
       "      <td>93.333333</td>\n",
       "      <td>87.5</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chrY_10165587</th>\n",
       "      <td>96.0</td>\n",
       "      <td>93.75</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.00</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>45.454545</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>95.454545</td>\n",
       "      <td>...</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>75.00</td>\n",
       "      <td>82.608696</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.0</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>93.333333</td>\n",
       "      <td>100.0</td>\n",
       "      <td>88.888889</td>\n",
       "      <td>90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chrY_10165621</th>\n",
       "      <td>88.0</td>\n",
       "      <td>100.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>66.666667</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>93.75</td>\n",
       "      <td>94.444444</td>\n",
       "      <td>72.727273</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>86.363636</td>\n",
       "      <td>...</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.00</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>93.333333</td>\n",
       "      <td>100.0</td>\n",
       "      <td>85.714286</td>\n",
       "      <td>93.333333</td>\n",
       "      <td>100.0</td>\n",
       "      <td>88.888889</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 255 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               M3204  M3503R  GX2467        DW33       M2402   M0806  \\\n",
       "index                                                                  \n",
       "chrY_10165553    0.0    6.25     NaN    0.000000    0.000000    0.00   \n",
       "chrY_10165561   92.0   81.25     NaN   83.333333   92.307692   87.50   \n",
       "chrY_10165579   80.0   68.75     NaN   33.333333   76.923077   93.75   \n",
       "chrY_10165587   96.0   93.75     NaN  100.000000  100.000000  100.00   \n",
       "chrY_10165621   88.0  100.00     NaN   66.666667  100.000000   93.75   \n",
       "\n",
       "                    M3405  B6D2F1_CR2705      GX2470  B6_CR2302  ...  \\\n",
       "index                                                            ...   \n",
       "chrY_10165553    0.000000       0.000000   14.285714   0.000000  ...   \n",
       "chrY_10165561   83.333333      63.636364   85.714286  90.909091  ...   \n",
       "chrY_10165579   88.888889      36.363636  100.000000  90.909091  ...   \n",
       "chrY_10165587  100.000000      45.454545  100.000000  95.454545  ...   \n",
       "chrY_10165621   94.444444      72.727273  100.000000  86.363636  ...   \n",
       "\n",
       "                    M0404   M2202       M2804         DW1  GX2519  \\\n",
       "index                                                               \n",
       "chrY_10165553   10.526316    6.25    4.347826    0.000000     0.0   \n",
       "chrY_10165561   89.473684   87.50   95.652174   73.333333    80.0   \n",
       "chrY_10165579   94.736842   68.75   86.956522   93.333333    60.0   \n",
       "chrY_10165587  100.000000   75.00   82.608696  100.000000   100.0   \n",
       "chrY_10165621  100.000000  100.00  100.000000   93.333333   100.0   \n",
       "\n",
       "               B6D2F1_CR2101      GX2481  GX2596       M0606  35D02  \n",
       "index                                                                \n",
       "chrY_10165553       0.000000    0.000000     0.0    0.000000   10.0  \n",
       "chrY_10165561      64.285714  100.000000   100.0   77.777778   70.0  \n",
       "chrY_10165579      64.285714   93.333333    87.5  100.000000   80.0  \n",
       "chrY_10165587     100.000000   93.333333   100.0   88.888889   90.0  \n",
       "chrY_10165621      85.714286   93.333333   100.0   88.888889  100.0  \n",
       "\n",
       "[5 rows x 255 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_path = Path.joinpath(cwd, project_dir, \"{}_coverage_0_binned_by_1.csv\".format(project_id))\n",
    "# data_path = Path.joinpath(cwd, project_dir, \"{}.csv\".format(project_id))\n",
    "print(data_path)\n",
    "\n",
    "# df_coverage = mtools.get_methylation_df(data_path, typ='coverage')\n",
    "df_percent = mtools.get_methylation_df(data_path, typ='percents')\n",
    "print(df_percent.shape)\n",
    "df_percent.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "17df8631-d505-4223-a3ab-18602fcb0ebe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>M3204</th>\n",
       "      <th>M3503R</th>\n",
       "      <th>GX2467</th>\n",
       "      <th>DW33</th>\n",
       "      <th>M2402</th>\n",
       "      <th>M0806</th>\n",
       "      <th>M3405</th>\n",
       "      <th>B6D2F1_CR2705</th>\n",
       "      <th>GX2470</th>\n",
       "      <th>B6_CR2302</th>\n",
       "      <th>...</th>\n",
       "      <th>M0404</th>\n",
       "      <th>M2202</th>\n",
       "      <th>M2804</th>\n",
       "      <th>DW1</th>\n",
       "      <th>GX2519</th>\n",
       "      <th>B6D2F1_CR2101</th>\n",
       "      <th>GX2481</th>\n",
       "      <th>GX2596</th>\n",
       "      <th>M0606</th>\n",
       "      <th>35D02</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>chr10_100015516</th>\n",
       "      <td>20.000000</td>\n",
       "      <td>8.333333</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>13.636364</td>\n",
       "      <td>15.789474</td>\n",
       "      <td>9.090909</td>\n",
       "      <td>19.230769</td>\n",
       "      <td>18.181818</td>\n",
       "      <td>3.846154</td>\n",
       "      <td>6.382979</td>\n",
       "      <td>...</td>\n",
       "      <td>4.651163</td>\n",
       "      <td>21.052632</td>\n",
       "      <td>18.181818</td>\n",
       "      <td>8.333333</td>\n",
       "      <td>21.428571</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>12.500000</td>\n",
       "      <td>11.764706</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr10_100015517</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.142857</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.555556</td>\n",
       "      <td>3.846154</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr10_100015540</th>\n",
       "      <td>10.666667</td>\n",
       "      <td>16.666667</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.250000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.846154</td>\n",
       "      <td>2.127660</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>19.047619</td>\n",
       "      <td>5.555556</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>12.500000</td>\n",
       "      <td>4.081633</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>11.764706</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr10_100015541</th>\n",
       "      <td>7.692308</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.090909</td>\n",
       "      <td>6.666667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>13.043478</td>\n",
       "      <td>3.571429</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>9.677419</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.882353</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>15.384615</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.555556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr10_100015553</th>\n",
       "      <td>12.318841</td>\n",
       "      <td>20.689655</td>\n",
       "      <td>5.714286</td>\n",
       "      <td>6.250000</td>\n",
       "      <td>2.439024</td>\n",
       "      <td>3.636364</td>\n",
       "      <td>9.230769</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.846154</td>\n",
       "      <td>...</td>\n",
       "      <td>2.325581</td>\n",
       "      <td>8.571429</td>\n",
       "      <td>7.692308</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>2.439024</td>\n",
       "      <td>17.142857</td>\n",
       "      <td>5.555556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chrY_998589</th>\n",
       "      <td>83.653846</td>\n",
       "      <td>72.000000</td>\n",
       "      <td>94.736842</td>\n",
       "      <td>39.285714</td>\n",
       "      <td>89.189189</td>\n",
       "      <td>89.583333</td>\n",
       "      <td>91.891892</td>\n",
       "      <td>57.142857</td>\n",
       "      <td>88.235294</td>\n",
       "      <td>79.761905</td>\n",
       "      <td>...</td>\n",
       "      <td>96.153846</td>\n",
       "      <td>88.095238</td>\n",
       "      <td>86.363636</td>\n",
       "      <td>66.666667</td>\n",
       "      <td>93.333333</td>\n",
       "      <td>57.692308</td>\n",
       "      <td>97.777778</td>\n",
       "      <td>83.333333</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>82.926829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chrY_998593</th>\n",
       "      <td>82.500000</td>\n",
       "      <td>90.476190</td>\n",
       "      <td>NaN</td>\n",
       "      <td>83.928571</td>\n",
       "      <td>96.774194</td>\n",
       "      <td>55.555556</td>\n",
       "      <td>95.238095</td>\n",
       "      <td>94.444444</td>\n",
       "      <td>78.571429</td>\n",
       "      <td>80.645161</td>\n",
       "      <td>...</td>\n",
       "      <td>82.352941</td>\n",
       "      <td>89.285714</td>\n",
       "      <td>91.304348</td>\n",
       "      <td>95.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>85.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>90.909091</td>\n",
       "      <td>88.888889</td>\n",
       "      <td>83.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chrY_998594</th>\n",
       "      <td>50.000000</td>\n",
       "      <td>44.000000</td>\n",
       "      <td>42.105263</td>\n",
       "      <td>92.857143</td>\n",
       "      <td>67.567568</td>\n",
       "      <td>72.916667</td>\n",
       "      <td>51.351351</td>\n",
       "      <td>52.380952</td>\n",
       "      <td>41.176471</td>\n",
       "      <td>51.190476</td>\n",
       "      <td>...</td>\n",
       "      <td>57.692308</td>\n",
       "      <td>73.809524</td>\n",
       "      <td>64.772727</td>\n",
       "      <td>66.666667</td>\n",
       "      <td>26.666667</td>\n",
       "      <td>76.923077</td>\n",
       "      <td>53.333333</td>\n",
       "      <td>58.333333</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>51.219512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chrY_998603</th>\n",
       "      <td>11.250000</td>\n",
       "      <td>7.142857</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.571429</td>\n",
       "      <td>6.250000</td>\n",
       "      <td>18.421053</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>10.526316</td>\n",
       "      <td>7.142857</td>\n",
       "      <td>8.196721</td>\n",
       "      <td>...</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>10.344828</td>\n",
       "      <td>4.347826</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.545455</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>37.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chrY_998604</th>\n",
       "      <td>87.500000</td>\n",
       "      <td>88.000000</td>\n",
       "      <td>88.888889</td>\n",
       "      <td>92.857143</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>85.714286</td>\n",
       "      <td>91.891892</td>\n",
       "      <td>72.727273</td>\n",
       "      <td>88.888889</td>\n",
       "      <td>87.058824</td>\n",
       "      <td>...</td>\n",
       "      <td>94.444444</td>\n",
       "      <td>95.348837</td>\n",
       "      <td>86.666667</td>\n",
       "      <td>96.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>80.769231</td>\n",
       "      <td>95.652174</td>\n",
       "      <td>76.923077</td>\n",
       "      <td>92.592593</td>\n",
       "      <td>85.365854</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1976056 rows × 255 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     M3204     M3503R     GX2467       DW33       M2402  \\\n",
       "index                                                                     \n",
       "chr10_100015516  20.000000   8.333333  10.000000  13.636364   15.789474   \n",
       "chr10_100015517   0.000000   0.000000   0.000000   0.000000    0.000000   \n",
       "chr10_100015540  10.666667  16.666667   5.000000   0.000000    0.000000   \n",
       "chr10_100015541   7.692308   0.000000   9.090909   6.666667    0.000000   \n",
       "chr10_100015553  12.318841  20.689655   5.714286   6.250000    2.439024   \n",
       "...                    ...        ...        ...        ...         ...   \n",
       "chrY_998589      83.653846  72.000000  94.736842  39.285714   89.189189   \n",
       "chrY_998593      82.500000  90.476190        NaN  83.928571   96.774194   \n",
       "chrY_998594      50.000000  44.000000  42.105263  92.857143   67.567568   \n",
       "chrY_998603      11.250000   7.142857        NaN   3.571429    6.250000   \n",
       "chrY_998604      87.500000  88.000000  88.888889  92.857143  100.000000   \n",
       "\n",
       "                     M0806      M3405  B6D2F1_CR2705     GX2470  B6_CR2302  \\\n",
       "index                                                                        \n",
       "chr10_100015516   9.090909  19.230769      18.181818   3.846154   6.382979   \n",
       "chr10_100015517   0.000000   7.142857       0.000000   0.000000   0.000000   \n",
       "chr10_100015540   6.250000   0.000000       0.000000   3.846154   2.127660   \n",
       "chr10_100015541  13.043478   3.571429       0.000000   0.000000   0.000000   \n",
       "chr10_100015553   3.636364   9.230769       0.000000   0.000000   3.846154   \n",
       "...                    ...        ...            ...        ...        ...   \n",
       "chrY_998589      89.583333  91.891892      57.142857  88.235294  79.761905   \n",
       "chrY_998593      55.555556  95.238095      94.444444  78.571429  80.645161   \n",
       "chrY_998594      72.916667  51.351351      52.380952  41.176471  51.190476   \n",
       "chrY_998603      18.421053   0.000000      10.526316   7.142857   8.196721   \n",
       "chrY_998604      85.714286  91.891892      72.727273  88.888889  87.058824   \n",
       "\n",
       "                 ...      M0404      M2202      M2804        DW1      GX2519  \\\n",
       "index            ...                                                           \n",
       "chr10_100015516  ...   4.651163  21.052632  18.181818   8.333333   21.428571   \n",
       "chr10_100015517  ...   0.000000   0.000000   0.000000   5.555556    3.846154   \n",
       "chr10_100015540  ...   0.000000   0.000000  19.047619   5.555556    0.000000   \n",
       "chr10_100015541  ...   9.677419   0.000000   5.882353   0.000000   15.384615   \n",
       "chr10_100015553  ...   2.325581   8.571429   7.692308   0.000000    0.000000   \n",
       "...              ...        ...        ...        ...        ...         ...   \n",
       "chrY_998589      ...  96.153846  88.095238  86.363636  66.666667   93.333333   \n",
       "chrY_998593      ...  82.352941  89.285714  91.304348  95.000000  100.000000   \n",
       "chrY_998594      ...  57.692308  73.809524  64.772727  66.666667   26.666667   \n",
       "chrY_998603      ...   8.000000  10.344828   4.347826   0.000000    0.000000   \n",
       "chrY_998604      ...  94.444444  95.348837  86.666667  96.000000  100.000000   \n",
       "\n",
       "                 B6D2F1_CR2101     GX2481     GX2596       M0606      35D02  \n",
       "index                                                                        \n",
       "chr10_100015516       0.000000   8.000000  12.500000   11.764706   5.000000  \n",
       "chr10_100015517       0.000000   0.000000   0.000000    0.000000   0.000000  \n",
       "chr10_100015540      12.500000   4.081633   0.000000   11.764706   0.000000  \n",
       "chr10_100015541       0.000000  10.000000   0.000000    0.000000   5.555556  \n",
       "chr10_100015553       0.000000   5.000000   2.439024   17.142857   5.555556  \n",
       "...                        ...        ...        ...         ...        ...  \n",
       "chrY_998589          57.692308  97.777778  83.333333  100.000000  82.926829  \n",
       "chrY_998593          85.000000        NaN  90.909091   88.888889  83.333333  \n",
       "chrY_998594          76.923077  53.333333  58.333333   50.000000  51.219512  \n",
       "chrY_998603           0.000000        NaN   4.545455    0.000000  37.500000  \n",
       "chrY_998604          80.769231  95.652174  76.923077   92.592593  85.365854  \n",
       "\n",
       "[1976056 rows x 255 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_percent.sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20af2299-3b2a-43c2-b412-b9884cb1d2d4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "195c0127-0afb-41c3-9de2-bd60b486b97d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/tarkhov/Dropbox/Harvard/Science/Stochastic aging/data/GSE80672/GSE80672_coverage_0_binned_by_1.csv\n",
      "(1976056, 255)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>M3204</th>\n",
       "      <th>M3503R</th>\n",
       "      <th>GX2467</th>\n",
       "      <th>DW33</th>\n",
       "      <th>M2402</th>\n",
       "      <th>M0806</th>\n",
       "      <th>M3405</th>\n",
       "      <th>B6D2F1_CR2705</th>\n",
       "      <th>GX2470</th>\n",
       "      <th>B6_CR2302</th>\n",
       "      <th>...</th>\n",
       "      <th>M0404</th>\n",
       "      <th>M2202</th>\n",
       "      <th>M2804</th>\n",
       "      <th>DW1</th>\n",
       "      <th>GX2519</th>\n",
       "      <th>B6D2F1_CR2101</th>\n",
       "      <th>GX2481</th>\n",
       "      <th>GX2596</th>\n",
       "      <th>M0606</th>\n",
       "      <th>35D02</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>chr10_100015516</th>\n",
       "      <td>20.000000</td>\n",
       "      <td>8.333333</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>13.636364</td>\n",
       "      <td>15.789474</td>\n",
       "      <td>9.090909</td>\n",
       "      <td>19.230769</td>\n",
       "      <td>18.181818</td>\n",
       "      <td>3.846154</td>\n",
       "      <td>6.382979</td>\n",
       "      <td>...</td>\n",
       "      <td>4.651163</td>\n",
       "      <td>21.052632</td>\n",
       "      <td>18.181818</td>\n",
       "      <td>8.333333</td>\n",
       "      <td>21.428571</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>12.500000</td>\n",
       "      <td>11.764706</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr10_100015517</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.142857</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.555556</td>\n",
       "      <td>3.846154</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr10_100015540</th>\n",
       "      <td>10.666667</td>\n",
       "      <td>16.666667</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.250000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.846154</td>\n",
       "      <td>2.127660</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>19.047619</td>\n",
       "      <td>5.555556</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>12.5</td>\n",
       "      <td>4.081633</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>11.764706</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr10_100015541</th>\n",
       "      <td>7.692308</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.090909</td>\n",
       "      <td>6.666667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>13.043478</td>\n",
       "      <td>3.571429</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>9.677419</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.882353</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>15.384615</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.555556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr10_100015553</th>\n",
       "      <td>12.318841</td>\n",
       "      <td>20.689655</td>\n",
       "      <td>5.714286</td>\n",
       "      <td>6.250000</td>\n",
       "      <td>2.439024</td>\n",
       "      <td>3.636364</td>\n",
       "      <td>9.230769</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.846154</td>\n",
       "      <td>...</td>\n",
       "      <td>2.325581</td>\n",
       "      <td>8.571429</td>\n",
       "      <td>7.692308</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>2.439024</td>\n",
       "      <td>17.142857</td>\n",
       "      <td>5.555556</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 255 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     M3204     M3503R     GX2467       DW33      M2402  \\\n",
       "index                                                                    \n",
       "chr10_100015516  20.000000   8.333333  10.000000  13.636364  15.789474   \n",
       "chr10_100015517   0.000000   0.000000   0.000000   0.000000   0.000000   \n",
       "chr10_100015540  10.666667  16.666667   5.000000   0.000000   0.000000   \n",
       "chr10_100015541   7.692308   0.000000   9.090909   6.666667   0.000000   \n",
       "chr10_100015553  12.318841  20.689655   5.714286   6.250000   2.439024   \n",
       "\n",
       "                     M0806      M3405  B6D2F1_CR2705    GX2470  B6_CR2302  \\\n",
       "index                                                                       \n",
       "chr10_100015516   9.090909  19.230769      18.181818  3.846154   6.382979   \n",
       "chr10_100015517   0.000000   7.142857       0.000000  0.000000   0.000000   \n",
       "chr10_100015540   6.250000   0.000000       0.000000  3.846154   2.127660   \n",
       "chr10_100015541  13.043478   3.571429       0.000000  0.000000   0.000000   \n",
       "chr10_100015553   3.636364   9.230769       0.000000  0.000000   3.846154   \n",
       "\n",
       "                 ...     M0404      M2202      M2804       DW1     GX2519  \\\n",
       "index            ...                                                        \n",
       "chr10_100015516  ...  4.651163  21.052632  18.181818  8.333333  21.428571   \n",
       "chr10_100015517  ...  0.000000   0.000000   0.000000  5.555556   3.846154   \n",
       "chr10_100015540  ...  0.000000   0.000000  19.047619  5.555556   0.000000   \n",
       "chr10_100015541  ...  9.677419   0.000000   5.882353  0.000000  15.384615   \n",
       "chr10_100015553  ...  2.325581   8.571429   7.692308  0.000000   0.000000   \n",
       "\n",
       "                 B6D2F1_CR2101     GX2481     GX2596      M0606     35D02  \n",
       "index                                                                      \n",
       "chr10_100015516            0.0   8.000000  12.500000  11.764706  5.000000  \n",
       "chr10_100015517            0.0   0.000000   0.000000   0.000000  0.000000  \n",
       "chr10_100015540           12.5   4.081633   0.000000  11.764706  0.000000  \n",
       "chr10_100015541            0.0  10.000000   0.000000   0.000000  5.555556  \n",
       "chr10_100015553            0.0   5.000000   2.439024  17.142857  5.555556  \n",
       "\n",
       "[5 rows x 255 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# data_path_binned = Path.joinpath(cwd, project_dir, \"{}_coverage_0_binned_by_1.csv\".format(project_id))\n",
    "data_path_binned = Path.joinpath(cwd, project_dir, \"{}_coverage_0_binned_by_10.csv\".format(project_id))\n",
    "print(data_path_binned)\n",
    "\n",
    "# df_coverage = mtools.get_methylation_df(data_path, typ='coverage')\n",
    "df_percent_binned = mtools.get_methylation_df(data_path_binned, typ='percents')\n",
    "print(df_percent_binned.shape)\n",
    "df_percent_binned.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0cd425a6-5843-4f9a-953d-062bc6c292df",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df_percent_binned' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [1]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mdf_percent_binned\u001b[49m\u001b[38;5;241m.\u001b[39msort_index()\n",
      "\u001b[0;31mNameError\u001b[0m: name 'df_percent_binned' is not defined"
     ]
    }
   ],
   "source": [
    "df_percent_binned.sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5fe6989-8641-40d5-9fea-414b509c82a4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "11f89ec9-2b21-4cdd-9fd6-ec1fe9f36a2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "ad = sc.AnnData(df_percent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "21c22243-b746-41d0-aa04-32997cd1cc97",
   "metadata": {},
   "outputs": [],
   "source": [
    "ad.var = pd.concat((ad.var, df_meta.loc[ad.var.index,:]), axis=1)\n",
    "ad.X = ad.X.astype(np.float16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "9927bb89-de9a-4e19-8d96-743062611f8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "ad.write_h5ad(Path.joinpath(project_dir, \"{}.h5ad\".format(project_id)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e2ab5369-2f79-4404-a579-22bafecf3ee3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/tarkhov/Dropbox/Harvard/Science/Stochastic aging/data/GSE80672/GSE80672_coverage_0_binned_by_1.h5ad\n"
     ]
    }
   ],
   "source": [
    "data_path = Path.joinpath(cwd, project_dir, \"{}_coverage_0_binned_by_1.h5ad\".format(project_id))\n",
    "print(data_path)\n",
    "ad = sc.read_h5ad(data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "030dcdf6-fa09-4d07-93cd-5e45a0f84960",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(510, 7)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sample Name</th>\n",
       "      <th>Strain/Condition</th>\n",
       "      <th>Age</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Prepared for Sequencing</th>\n",
       "      <th>Used in Subset 1/2</th>\n",
       "      <th>Data Type</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Name PerCov</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20D02_percentage</th>\n",
       "      <td>20D02</td>\n",
       "      <td>C57BL/6_NIA/BWH</td>\n",
       "      <td>0.669922</td>\n",
       "      <td>Male</td>\n",
       "      <td>D.A. Petkovich</td>\n",
       "      <td>NaN</td>\n",
       "      <td>percentage</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20D03_percentage</th>\n",
       "      <td>20D03</td>\n",
       "      <td>C57BL/6_NIA/BWH</td>\n",
       "      <td>0.669922</td>\n",
       "      <td>Male</td>\n",
       "      <td>D.A. Petkovich</td>\n",
       "      <td>NaN</td>\n",
       "      <td>percentage</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20D04_percentage</th>\n",
       "      <td>20D04</td>\n",
       "      <td>C57BL/6_NIA/BWH</td>\n",
       "      <td>0.669922</td>\n",
       "      <td>Male</td>\n",
       "      <td>D.A. Petkovich</td>\n",
       "      <td>NaN</td>\n",
       "      <td>percentage</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20D05_percentage</th>\n",
       "      <td>20D05</td>\n",
       "      <td>C57BL/6_NIA/BWH</td>\n",
       "      <td>0.669922</td>\n",
       "      <td>Male</td>\n",
       "      <td>D.A. Petkovich</td>\n",
       "      <td>NaN</td>\n",
       "      <td>percentage</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20D06_percentage</th>\n",
       "      <td>20D06</td>\n",
       "      <td>C57BL/6_NIA/BWH</td>\n",
       "      <td>0.669922</td>\n",
       "      <td>Male</td>\n",
       "      <td>D.A. Petkovich</td>\n",
       "      <td>NaN</td>\n",
       "      <td>percentage</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Sample Name Strain/Condition       Age   Sex  \\\n",
       "Name PerCov                                                     \n",
       "20D02_percentage       20D02  C57BL/6_NIA/BWH  0.669922  Male   \n",
       "20D03_percentage       20D03  C57BL/6_NIA/BWH  0.669922  Male   \n",
       "20D04_percentage       20D04  C57BL/6_NIA/BWH  0.669922  Male   \n",
       "20D05_percentage       20D05  C57BL/6_NIA/BWH  0.669922  Male   \n",
       "20D06_percentage       20D06  C57BL/6_NIA/BWH  0.669922  Male   \n",
       "\n",
       "                 Prepared for Sequencing  Used in Subset 1/2   Data Type  \n",
       "Name PerCov                                                               \n",
       "20D02_percentage           D.A. Petkovich                NaN  percentage  \n",
       "20D03_percentage           D.A. Petkovich                NaN  percentage  \n",
       "20D04_percentage           D.A. Petkovich                NaN  percentage  \n",
       "20D05_percentage           D.A. Petkovich                NaN  percentage  \n",
       "20D06_percentage           D.A. Petkovich                NaN  percentage  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_meta.reset_index(inplace=True)\n",
    "df_meta_2 = df_meta.copy()\n",
    "df_meta['Name PerCov'] = df_meta['Sample Name'].str.cat(['percentage'] * df_meta.shape[0], sep='_')\n",
    "df_meta['Data Type'] = 'percentage'\n",
    "df_meta_2['Name PerCov'] = df_meta_2['Sample Name'].str.cat(['coverage'] * df_meta.shape[0], sep='_')\n",
    "df_meta_2['Data Type'] = 'coverage'\n",
    "df_meta['Name PerCov'] = df_meta['Sample Name'].str.cat(['percentage'] * df_meta.shape[0], sep='_')\n",
    "df_meta = pd.concat((df_meta, df_meta_2), axis=0)\n",
    "df_meta.set_index('Name PerCov', inplace=True)\n",
    "print(df_meta.shape)\n",
    "df_meta.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f7ba8a12-35c1-4357-a7c2-63f0be9cc701",
   "metadata": {},
   "outputs": [],
   "source": [
    "ad.var = pd.concat((ad.var, df_meta.loc[ad.var.index,:]), axis=1)\n",
    "ad.X = ad.X.astype(np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7da45391-eaa0-4dc3-896b-e895e7814ed3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Sample Name' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Data Type' as categorical\n"
     ]
    }
   ],
   "source": [
    "ad.write_h5ad(Path.joinpath(project_dir, \"{}_coverage_0_binned_by_1_wmeta.h5ad\".format(project_id)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "ae07a904-dff4-41de-91eb-eef9b8d42bbb",
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "'<' not supported between instances of 'AnnData' and 'int'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [27]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m ad_bulk \u001b[38;5;241m=\u001b[39m ad[:,::\u001b[38;5;241m2\u001b[39m]\n\u001b[1;32m      2\u001b[0m ad_bulk\u001b[38;5;241m.\u001b[39mX \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39marray(ad_bulk\u001b[38;5;241m.\u001b[39mX\u001b[38;5;241m.\u001b[39mtodense())\n\u001b[0;32m----> 3\u001b[0m ad_bulk\u001b[38;5;241m.\u001b[39mX[\u001b[43mad\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[43m,\u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m:\u001b[49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m<\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m] \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mnan\n\u001b[1;32m      4\u001b[0m ad_bulk\u001b[38;5;241m.\u001b[39mX\n",
      "\u001b[0;31mTypeError\u001b[0m: '<' not supported between instances of 'AnnData' and 'int'"
     ]
    }
   ],
   "source": [
    "ad_bulk = ad[:,::2]\n",
    "ad_bulk.X = np.array(ad_bulk.X.todense())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "c4db0c7f-ea1c-4cd6-adc4-86bc302574fa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[20.       ,  8.333333 , 10.       , ..., 12.5      , 11.764706 ,\n",
       "         5.       ],\n",
       "       [ 0.       ,        nan,  0.       , ...,        nan,  0.       ,\n",
       "         0.       ],\n",
       "       [10.666667 , 16.666666 ,  5.       , ...,  0.       , 11.764706 ,\n",
       "         0.       ],\n",
       "       ...,\n",
       "       [50.       , 44.       , 42.105263 , ..., 58.333332 , 50.       ,\n",
       "        51.219513 ],\n",
       "       [11.25     ,  7.142857 ,        nan, ...,  4.5454545,  0.       ,\n",
       "        37.5      ],\n",
       "       [87.5      , 88.       , 88.888885 , ..., 76.92308  , 92.59259  ,\n",
       "        85.36585  ]], dtype=float32)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ad_bulk.X[np.array(ad[:,1::2].X.todense()) < 10] = np.nan\n",
    "ad_bulk.X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "ff7bd935-499b-4f02-b951-984b6e077b94",
   "metadata": {},
   "outputs": [],
   "source": [
    "# plt.hist(np.array(ad[:10,1::2].X.todense()).flatten())\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "67c28df6-f536-4a9c-96b2-a6512b34cb02",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[20.       ,  8.333333 , 10.       , ..., 12.5      , 11.764706 ,\n",
       "         5.       ],\n",
       "       [ 0.       ,        nan,  0.       , ...,        nan,  0.       ,\n",
       "         0.       ],\n",
       "       [10.666667 , 16.666666 ,  5.       , ...,  0.       , 11.764706 ,\n",
       "         0.       ],\n",
       "       ...,\n",
       "       [50.       , 44.       , 42.105263 , ..., 58.333332 , 50.       ,\n",
       "        51.219513 ],\n",
       "       [11.25     ,  7.142857 ,        nan, ...,  4.5454545,  0.       ,\n",
       "        37.5      ],\n",
       "       [87.5      , 88.       , 88.888885 , ..., 76.92308  , 92.59259  ,\n",
       "        85.36585  ]], dtype=float32)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ad_bulk.X = np.array(ad_bulk.X.todense())\n",
    "ad_bulk.X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "dbf06210-2ba9-40b9-858b-aa8cd950e65d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEDCAYAAAAlRP8qAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAOj0lEQVR4nO3df4zkdX3H8eer3KlVtGhvW+iBHG1IVUwQckHQpiFqU36l1z9sciT+iDG5aLTVxsSgNtr6l01aUxWFXJUi1kBaJXhR/BU1AZNKXU5E4KRehcoVKqtEkGqqF9/9Y75Xt8vuzuzd7M3t+56PZLIz3+9nZz6f3N0zX77znSFVhSRp4/uVWU9AkjQdBl2SmjDoktSEQZekJgy6JDVh0CWpiZkGPck1SR5OctcEY5+d5CtJvpHkziSXHI05StJGMesj9GuBiyYc+xfAP1XVOcBO4EPrNSlJ2ohmGvSqugV4ZPG2JL+T5HNJbk9ya5LnHBoOPGO4/2vAg0dxqpJ0zNs06wksYzfwuqr6TpIXMjoSfwnwl8AXkvwp8DTgZbOboiQde46poCc5EXgR8M9JDm1+8vDzcuDaqvrbJBcAH0vy/Kr6xQymKknHnGMq6IxOAf2oql6wzL7XMpxvr6p/SfIUYAvw8NGbniQdu2b9puj/U1WPAfcl+ROAjJw97P4e8NJh+3OBpwALM5moJB2DMstvW0xyPXAhoyPt7wPvAr4MXAWcAmwGbqiqdyd5HvD3wImM3iB9a1V9YRbzlqRj0UyDLkmanmPqlIsk6fCNfVM0yWnAdcDJwC+A3VX1viVjLgQ+Bdw3bLqxqt692vNu2bKltm3btvYZS9Jx7Pbbb/9BVc0tt2+Sq1wOAm+pqr1Jng7cnuSLVXXPknG3VtVlk05q27ZtzM/PTzpckgQk+Y+V9o095VJVD1XV3uH+j4F9wNbpTU+SNA1rOoeeZBtwDnDbMrsvSPLNJJ9NctYKv78ryXyS+YUFrziUpGmaOOjDpzg/Cbx5uF58sb3A6VV1NvAB4KblnqOqdlfV9qraPje37CkgSdJhmijoSTYzivnHq+rGpfur6rGqeny4fzOwOcmWqc5UkrSqsUHP6EtVPgLsq6r3rjDm5GEcSc4bnveH05yoJGl1k1zl8mLglcC3ktwxbHs78GyAqroaeDnw+iQHgZ8CO8tPLEnSUTU26FX1VSBjxlwJXDmtSUmS1s5PikpSEwZdkpo41r4PfSLbrvjMzF77/vdcOrPXlqTVeIQuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNTE26ElOS/KVJPuS3J3kTcuMSZL3J9mf5M4k567PdCVJK9k0wZiDwFuqam+SpwO3J/liVd2zaMzFwJnD7YXAVcNPSdJRMvYIvaoeqqq9w/0fA/uArUuG7QCuq5GvASclOWXqs5UkrWhN59CTbAPOAW5bsmsr8MCixwd4YvRJsivJfJL5hYWFNU5VkrSaiYOe5ETgk8Cbq+qxpbuX+ZV6woaq3VW1vaq2z83NrW2mkqRVTRT0JJsZxfzjVXXjMkMOAKctenwq8OCRT0+SNKlJrnIJ8BFgX1W9d4Vhe4BXDVe7nA88WlUPTXGekqQxJrnK5cXAK4FvJblj2PZ24NkAVXU1cDNwCbAf+AnwmqnPVJK0qrFBr6qvsvw58sVjCnjDtCYlSVo7PykqSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWpibNCTXJPk4SR3rbD/wiSPJrljuL1z+tOUJI2zaYIx1wJXAtetMubWqrpsKjOSJB2WsUfoVXUL8MhRmIsk6QhM6xz6BUm+meSzSc5aaVCSXUnmk8wvLCxM6aUlSTCdoO8FTq+qs4EPADetNLCqdlfV9qraPjc3N4WXliQdcsRBr6rHqurx4f7NwOYkW454ZpKkNTnioCc5OUmG++cNz/nDI31eSdLajL3KJcn1wIXAliQHgHcBmwGq6mrg5cDrkxwEfgrsrKpatxlLkpY1NuhVdfmY/VcyuqxRkjRDflJUkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktTE2KAnuSbJw0nuWmF/krw/yf4kdyY5d/rTlCSNM8kR+rXARavsvxg4c7jtAq468mlJktZqbNCr6hbgkVWG7ACuq5GvASclOWVaE5QkTWYa59C3Ag8senxg2PYESXYlmU8yv7CwMIWXliQdMo2gZ5lttdzAqtpdVduravvc3NwUXlqSdMg0gn4AOG3R41OBB6fwvJKkNZhG0PcArxqudjkfeLSqHprC80qS1mDTuAFJrgcuBLYkOQC8C9gMUFVXAzcDlwD7gZ8Ar1mvyUqSVjY26FV1+Zj9BbxhajOSJB2WsUGXpI62XfGZmb32/e+5dF2e14/+S1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMTBT3JRUnuTbI/yRXL7L8wyaNJ7hhu75z+VCVJq9k0bkCSE4APAn8AHAC+nmRPVd2zZOitVXXZOsxRkjSBSY7QzwP2V9V3q+pnwA3AjvWdliRprSYJ+lbggUWPDwzblrogyTeTfDbJWcs9UZJdSeaTzC8sLBzGdCVJK5kk6FlmWy15vBc4varOBj4A3LTcE1XV7qraXlXb5+bm1jRRSdLqJgn6AeC0RY9PBR5cPKCqHquqx4f7NwObk2yZ2iwlSWNNEvSvA2cmOSPJk4CdwJ7FA5KcnCTD/fOG5/3htCcrSVrZ2KtcqupgkjcCnwdOAK6pqruTvG7YfzXwcuD1SQ4CPwV2VtXS0zKSpHU0Nujwf6dRbl6y7epF968Erpzu1CRJa+EnRSWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTUz00X/90rYrPjOT173/PZfO5HUlbRweoUtSEwZdkpow6JLUhEGXpCZ8U1TSTM3qQoOOPEKXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDXhR/83iFl+PNrvYu/Pj9/34BG6JDXhEbp0DPFIWUfCoGus4/F/u2dYtREZdB2zjKq0Np5Dl6QmDLokNWHQJamJiYKe5KIk9ybZn+SKZfYnyfuH/XcmOXf6U5UkrWZs0JOcAHwQuBh4HnB5kuctGXYxcOZw2wVcNeV5SpLGmOQI/Txgf1V9t6p+BtwA7FgyZgdwXY18DTgpySlTnqskaRWTXLa4FXhg0eMDwAsnGLMVeGjxoCS7GB3BAzye5N41zfaXtgA/OMzf3ahc8/HBNR8H8tdHtObTV9oxSdCzzLY6jDFU1W5g9wSvufqEkvmq2n6kz7ORuObjg2s+PqzXmic55XIAOG3R41OBBw9jjCRpHU0S9K8DZyY5I8mTgJ3AniVj9gCvGq52OR94tKoeWvpEkqT1M/aUS1UdTPJG4PPACcA1VXV3ktcN+68GbgYuAfYDPwFes35TBqZw2mYDcs3HB9d8fFiXNafqCae6JUkbkJ8UlaQmDLokNbHhgj7uawg6SHJakq8k2Zfk7iRvGrY/K8kXk3xn+PnMWc91mpKckOQbST49PO6+3pOSfCLJt4c/6wuOgzX/+fB3+q4k1yd5Src1J7kmycNJ7lq0bcU1Jnnb0LN7k/zhkbz2hgr6hF9D0MFB4C1V9VzgfOANwzqvAL5UVWcCXxoed/ImYN+ix93X+z7gc1X1HOBsRmtvu+YkW4E/A7ZX1fMZXWSxk35rvha4aMm2Zdc4/LveCZw1/M6Hhs4dlg0VdCb7GoINr6oeqqq9w/0fM/qHvpXRWj86DPso8MczmeA6SHIqcCnw4UWbO6/3GcDvAx8BqKqfVdWPaLzmwSbgV5NsAp7K6PMqrdZcVbcAjyzZvNIadwA3VNX/VNV9jK4UPO9wX3ujBX2lrxhoK8k24BzgNuA3D13fP/z8jRlObdr+Dngr8ItF2zqv97eBBeAfhtNMH07yNBqvuar+E/gb4HuMvhbk0ar6Ao3XvMhKa5xq0zZa0Cf6ioEukpwIfBJ4c1U9Nuv5rJcklwEPV9Xts57LUbQJOBe4qqrOAf6bjX+qYVXDeeMdwBnAbwFPS/KK2c5q5qbatI0W9OPmKwaSbGYU849X1Y3D5u8f+hbL4efDs5rflL0Y+KMk9zM6jfaSJP9I3/XC6O/ygaq6bXj8CUaB77zmlwH3VdVCVf0cuBF4Eb3XfMhKa5xq0zZa0Cf5GoINL0kYnVvdV1XvXbRrD/Dq4f6rgU8d7bmth6p6W1WdWlXbGP2ZfrmqXkHT9QJU1X8BDyT53WHTS4F7aLxmRqdazk/y1OHv+EsZvT/Uec2HrLTGPcDOJE9Ocgaj/6fEvx72q1TVhrox+oqBfwP+HXjHrOezTmv8PUb/2XUncMdwuwT4dUbvkH9n+PmsWc91HdZ+IfDp4X7r9QIvAOaHP+ebgGceB2v+K+DbwF3Ax4And1szcD2j9wh+zugI/LWrrRF4x9Cze4GLj+S1/ei/JDWx0U65SJJWYNAlqQmDLklNGHRJasKgS1ITBl2SmjDoktTE/wKrG5p57CsaAQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(ad_bulk.X.flatten())\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "1c0a41b8-c093-4b0b-ab0d-77de5726fdcf",
   "metadata": {},
   "outputs": [],
   "source": [
    "ad_bulk.write_h5ad(Path.joinpath(project_dir, \"{}_coverage_0_binned_by_1_wmeta_only_percent.h5ad\".format(project_id)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "40b4669d-fd5c-4f76-bfa0-eaf451be38ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Sample Name' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Data Type' as categorical\n"
     ]
    }
   ],
   "source": [
    "ad.write_h5ad(Path.joinpath(project_dir, \"{}_wmeta.h5ad\".format(project_id)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a962c77-f8d4-4c98-b7c9-4aab7d133580",
   "metadata": {},
   "source": [
    "# Human methylation data — GSE40279 (Hannum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "a5dc48d1-f9c8-4779-864b-500516b312c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "cwd = Path.cwd()\n",
    "project_dir = Path.joinpath(cwd, Path('data/GSE40279/'))\n",
    "project_id = 'GSE40279'\n",
    "project_samples_dir = 'GSE40279_RAW'\n",
    "\n",
    "metainfo_old_path = 'sample.csv'\n",
    "metainfo_path = 'Hannum_Metainfo.csv'\n",
    "gl_metainfo_old_path = Path.joinpath(project_dir, metainfo_old_path)\n",
    "gl_metainfo_path = Path.joinpath(project_dir, metainfo_path)\n",
    "data_path = Path.joinpath(cwd, project_dir, \"{}.csv\".format(project_id))\n",
    "gl_project_dir = Path.joinpath(cwd, project_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "ffb625af-204d-48ff-8364-88f9fe9722b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accession</th>\n",
       "      <th>Age</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>X1001</th>\n",
       "      <td>GSM989827</td>\n",
       "      <td>67.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>X1002</th>\n",
       "      <td>GSM989828</td>\n",
       "      <td>89.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>X1003</th>\n",
       "      <td>GSM989829</td>\n",
       "      <td>66.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>X1004</th>\n",
       "      <td>GSM989830</td>\n",
       "      <td>64.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>X1005</th>\n",
       "      <td>GSM989831</td>\n",
       "      <td>62.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Accession   Age\n",
       "ID                    \n",
       "X1001  GSM989827  67.0\n",
       "X1002  GSM989828  89.0\n",
       "X1003  GSM989829  66.0\n",
       "X1004  GSM989830  64.0\n",
       "X1005  GSM989831  62.0"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_meta = pd.read_csv(gl_metainfo_old_path)\n",
    "df_meta['Age'] = df_meta['Title'].apply(lambda x: float(x.split(' ')[1].replace('y', '')))\n",
    "df_meta['ID'] = df_meta['Title'].apply(lambda x: \"X{}\".format(x.split(' ')[2]))\n",
    "df_meta.set_index('ID', inplace=True)\n",
    "df_meta.drop([\"Title\", \"Supplementary Types\", \"Supplementary Links\", \"SRA Accession\", \"Contact\", \"Release Date\", \"Channels\", \"Taxonomy\", \"Platform\", \"Series\", \"Sample Type\"], axis=1, inplace=True)\n",
    "df_meta[\"Accession\"] = df_meta[\"Accession\"].astype(\"category\")\n",
    "df_meta[\"Age\"] = df_meta[\"Age\"].astype(np.float16)\n",
    "df_meta.to_csv(gl_metainfo_path)\n",
    "df_meta.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "1a9a3519-88d8-40df-8866-ae103a81e63c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(473034, 656)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X1001</th>\n",
       "      <th>X1002</th>\n",
       "      <th>X1003</th>\n",
       "      <th>X1004</th>\n",
       "      <th>X1005</th>\n",
       "      <th>X1006</th>\n",
       "      <th>X1007</th>\n",
       "      <th>X1008</th>\n",
       "      <th>X1009</th>\n",
       "      <th>X1010</th>\n",
       "      <th>...</th>\n",
       "      <th>X11085</th>\n",
       "      <th>X11086</th>\n",
       "      <th>X11087</th>\n",
       "      <th>X11089</th>\n",
       "      <th>X11090</th>\n",
       "      <th>X11092</th>\n",
       "      <th>X11093</th>\n",
       "      <th>X11094</th>\n",
       "      <th>X11095</th>\n",
       "      <th>X11096</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_REF</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cg13869341</th>\n",
       "      <td>0.849261</td>\n",
       "      <td>0.897434</td>\n",
       "      <td>0.751596</td>\n",
       "      <td>0.871313</td>\n",
       "      <td>0.775703</td>\n",
       "      <td>0.905383</td>\n",
       "      <td>0.809341</td>\n",
       "      <td>0.831056</td>\n",
       "      <td>0.785725</td>\n",
       "      <td>0.851040</td>\n",
       "      <td>...</td>\n",
       "      <td>0.872390</td>\n",
       "      <td>0.797936</td>\n",
       "      <td>0.813616</td>\n",
       "      <td>0.838759</td>\n",
       "      <td>0.845707</td>\n",
       "      <td>0.908023</td>\n",
       "      <td>0.800924</td>\n",
       "      <td>0.781394</td>\n",
       "      <td>0.844790</td>\n",
       "      <td>0.863501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg14008030</th>\n",
       "      <td>0.505916</td>\n",
       "      <td>0.476842</td>\n",
       "      <td>0.487245</td>\n",
       "      <td>0.466692</td>\n",
       "      <td>0.490255</td>\n",
       "      <td>0.536831</td>\n",
       "      <td>0.513063</td>\n",
       "      <td>0.517671</td>\n",
       "      <td>0.526291</td>\n",
       "      <td>0.482147</td>\n",
       "      <td>...</td>\n",
       "      <td>0.609596</td>\n",
       "      <td>0.589514</td>\n",
       "      <td>0.550069</td>\n",
       "      <td>0.536344</td>\n",
       "      <td>0.573834</td>\n",
       "      <td>0.555264</td>\n",
       "      <td>0.578522</td>\n",
       "      <td>0.551470</td>\n",
       "      <td>0.570779</td>\n",
       "      <td>0.620469</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg12045430</th>\n",
       "      <td>0.072590</td>\n",
       "      <td>0.079020</td>\n",
       "      <td>0.089230</td>\n",
       "      <td>0.076666</td>\n",
       "      <td>0.079986</td>\n",
       "      <td>0.096142</td>\n",
       "      <td>0.084693</td>\n",
       "      <td>0.084978</td>\n",
       "      <td>0.076284</td>\n",
       "      <td>0.088784</td>\n",
       "      <td>...</td>\n",
       "      <td>0.074361</td>\n",
       "      <td>0.042380</td>\n",
       "      <td>0.047387</td>\n",
       "      <td>0.036802</td>\n",
       "      <td>0.025272</td>\n",
       "      <td>0.051956</td>\n",
       "      <td>0.063291</td>\n",
       "      <td>0.041310</td>\n",
       "      <td>0.059926</td>\n",
       "      <td>0.018954</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg20826792</th>\n",
       "      <td>0.186961</td>\n",
       "      <td>0.228201</td>\n",
       "      <td>0.237660</td>\n",
       "      <td>0.253624</td>\n",
       "      <td>0.220404</td>\n",
       "      <td>0.229790</td>\n",
       "      <td>0.221750</td>\n",
       "      <td>0.264803</td>\n",
       "      <td>0.198577</td>\n",
       "      <td>0.248889</td>\n",
       "      <td>...</td>\n",
       "      <td>0.187167</td>\n",
       "      <td>0.163056</td>\n",
       "      <td>0.187181</td>\n",
       "      <td>0.117650</td>\n",
       "      <td>0.122737</td>\n",
       "      <td>0.197111</td>\n",
       "      <td>0.223854</td>\n",
       "      <td>0.188288</td>\n",
       "      <td>0.129233</td>\n",
       "      <td>0.051887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00381604</th>\n",
       "      <td>0.036803</td>\n",
       "      <td>0.053161</td>\n",
       "      <td>0.045588</td>\n",
       "      <td>0.032824</td>\n",
       "      <td>0.035804</td>\n",
       "      <td>0.018335</td>\n",
       "      <td>0.046591</td>\n",
       "      <td>0.045572</td>\n",
       "      <td>0.049980</td>\n",
       "      <td>0.046218</td>\n",
       "      <td>...</td>\n",
       "      <td>0.011383</td>\n",
       "      <td>0.004570</td>\n",
       "      <td>0.014957</td>\n",
       "      <td>0.012622</td>\n",
       "      <td>0.015449</td>\n",
       "      <td>0.041821</td>\n",
       "      <td>0.038828</td>\n",
       "      <td>0.022204</td>\n",
       "      <td>0.013555</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 656 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               X1001     X1002     X1003     X1004     X1005     X1006  \\\n",
       "ID_REF                                                                   \n",
       "cg13869341  0.849261  0.897434  0.751596  0.871313  0.775703  0.905383   \n",
       "cg14008030  0.505916  0.476842  0.487245  0.466692  0.490255  0.536831   \n",
       "cg12045430  0.072590  0.079020  0.089230  0.076666  0.079986  0.096142   \n",
       "cg20826792  0.186961  0.228201  0.237660  0.253624  0.220404  0.229790   \n",
       "cg00381604  0.036803  0.053161  0.045588  0.032824  0.035804  0.018335   \n",
       "\n",
       "               X1007     X1008     X1009     X1010  ...    X11085    X11086  \\\n",
       "ID_REF                                              ...                       \n",
       "cg13869341  0.809341  0.831056  0.785725  0.851040  ...  0.872390  0.797936   \n",
       "cg14008030  0.513063  0.517671  0.526291  0.482147  ...  0.609596  0.589514   \n",
       "cg12045430  0.084693  0.084978  0.076284  0.088784  ...  0.074361  0.042380   \n",
       "cg20826792  0.221750  0.264803  0.198577  0.248889  ...  0.187167  0.163056   \n",
       "cg00381604  0.046591  0.045572  0.049980  0.046218  ...  0.011383  0.004570   \n",
       "\n",
       "              X11087    X11089    X11090    X11092    X11093    X11094  \\\n",
       "ID_REF                                                                   \n",
       "cg13869341  0.813616  0.838759  0.845707  0.908023  0.800924  0.781394   \n",
       "cg14008030  0.550069  0.536344  0.573834  0.555264  0.578522  0.551470   \n",
       "cg12045430  0.047387  0.036802  0.025272  0.051956  0.063291  0.041310   \n",
       "cg20826792  0.187181  0.117650  0.122737  0.197111  0.223854  0.188288   \n",
       "cg00381604  0.014957  0.012622  0.015449  0.041821  0.038828  0.022204   \n",
       "\n",
       "              X11095    X11096  \n",
       "ID_REF                          \n",
       "cg13869341  0.844790  0.863501  \n",
       "cg14008030  0.570779  0.620469  \n",
       "cg12045430  0.059926  0.018954  \n",
       "cg20826792  0.129233  0.051887  \n",
       "cg00381604  0.013555  0.000000  \n",
       "\n",
       "[5 rows x 656 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_full = pd.read_csv(Path.joinpath(project_dir, \"GSE40279_average_beta.txt\"), sep='\\t', index_col=0)\n",
    "print(df_full.shape)\n",
    "df_full.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "307346b0-b7b6-4051-bb91-2ccf6b9db322",
   "metadata": {},
   "outputs": [],
   "source": [
    "ad = sc.AnnData(df_full)\n",
    "ad.var = pd.concat((ad.var, df_meta.loc[ad.var.index,:]), axis=1)\n",
    "ad.X = ad.X.astype(np.float16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "5736d117-8c21-4ebb-97fa-5215fb1ada7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "ad.write_h5ad(Path.joinpath(project_dir, \"{}.h5ad\".format(project_id)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d998023-53ff-41d6-b4f0-a57d845ba381",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "e6163baa-793c-48b1-a88a-3b5ffe4ebcf4",
   "metadata": {},
   "source": [
    "# Mice embryos GSE34864"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "99d360d9-cbf6-40a3-8eaa-0431170e0247",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import pwd\n",
    "from pathlib import Path\n",
    "from tqdm import tqdm, trange\n",
    "from tqdm.contrib.concurrent import process_map\n",
    "import meth.tools as mtools\n",
    "import numpy as np\n",
    "\n",
    "cwd = Path.cwd()\n",
    "project_dir = Path('data/GSE34864')\n",
    "project_id = 'GSE34864'\n",
    "project_samples_dir = 'GSE34864_RAW'\n",
    "path_samples = Path.joinpath(project_dir, project_samples_dir)\n",
    "gl_path_samples = Path.joinpath(cwd, path_samples)\n",
    "gl_project_dir = Path.joinpath(cwd, project_dir)\n",
    "all_samples = os.listdir(gl_path_samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e791b12d-b173-4631-bd26-8de9bf9d968e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/tarkhov/Dropbox/Harvard/Science/Stochastic aging/data/GSE34864/GSE34864.csv\n",
      "(1533043, 76)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GSM856541_ICM-BDF1-AJ-3a</th>\n",
       "      <th>GSM856503_zygote-BDF1-X1-3</th>\n",
       "      <th>GSM856551_E75-BDF1-BDF1-2</th>\n",
       "      <th>GSM856537_8cellNoPB2-BDF1-X1-2</th>\n",
       "      <th>GSM856495_oocytes-BDF1-1</th>\n",
       "      <th>GSM856516_2cellNoPB2-BDF1-X1-3</th>\n",
       "      <th>GSM856508_2cell-BDF1-BDF1-1</th>\n",
       "      <th>GSM856530_8cell-BDF1-BDF1-3</th>\n",
       "      <th>GSM856550_E75-BDF1-BDF1-1</th>\n",
       "      <th>GSM870007_sperm-X1-3</th>\n",
       "      <th>...</th>\n",
       "      <th>GSM856526_4cellNoPB2-BDF1-X1-2</th>\n",
       "      <th>GSM856502_zygote-BDF1-BDF1-2</th>\n",
       "      <th>GSM856507_zygoteNoPB-BDF1-X1-3</th>\n",
       "      <th>GSM856535_8cellNoPB-BDF1-X1-2</th>\n",
       "      <th>GSM856525_4cellNoPB2-BDF1-X1-1</th>\n",
       "      <th>GSM870002_zygoteNoPB2-BDF1-X1-5</th>\n",
       "      <th>GSM856509_2cell-BDF1-X1-2</th>\n",
       "      <th>GSM856515_2cellNoPB2-BDF1-X1-2</th>\n",
       "      <th>GSM856556_Brain-2</th>\n",
       "      <th>GSM856505_zygoteNoPB-BDF1-X1-1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>chr1_3010894</th>\n",
       "      <td>0.571429</td>\n",
       "      <td>0.260000</td>\n",
       "      <td>0.711111</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.562500</td>\n",
       "      <td>0.870968</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.827586</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr1_3010922</th>\n",
       "      <td>0.794872</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.818182</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.917647</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr1_3089715</th>\n",
       "      <td>0.300000</td>\n",
       "      <td>0.370370</td>\n",
       "      <td>0.765957</td>\n",
       "      <td>0.340000</td>\n",
       "      <td>0.377778</td>\n",
       "      <td>0.527778</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.389831</td>\n",
       "      <td>0.846154</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.426230</td>\n",
       "      <td>0.324324</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.409091</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.5625</td>\n",
       "      <td>0.936170</td>\n",
       "      <td>0.435897</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr1_3089865</th>\n",
       "      <td>0.227273</td>\n",
       "      <td>0.446154</td>\n",
       "      <td>0.580000</td>\n",
       "      <td>0.580645</td>\n",
       "      <td>0.563636</td>\n",
       "      <td>0.466667</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.813953</td>\n",
       "      <td>0.823529</td>\n",
       "      <td>...</td>\n",
       "      <td>0.424242</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.509434</td>\n",
       "      <td>0.446809</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.409836</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.525000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr1_3102314</th>\n",
       "      <td>0.548387</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 76 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              GSM856541_ICM-BDF1-AJ-3a  GSM856503_zygote-BDF1-X1-3  \\\n",
       "index                                                                \n",
       "chr1_3010894                  0.571429                    0.260000   \n",
       "chr1_3010922                  0.794872                         NaN   \n",
       "chr1_3089715                  0.300000                    0.370370   \n",
       "chr1_3089865                  0.227273                    0.446154   \n",
       "chr1_3102314                  0.548387                         NaN   \n",
       "\n",
       "              GSM856551_E75-BDF1-BDF1-2  GSM856537_8cellNoPB2-BDF1-X1-2  \\\n",
       "index                                                                     \n",
       "chr1_3010894                   0.711111                             NaN   \n",
       "chr1_3010922                        NaN                             NaN   \n",
       "chr1_3089715                   0.765957                        0.340000   \n",
       "chr1_3089865                   0.580000                        0.580645   \n",
       "chr1_3102314                        NaN                             NaN   \n",
       "\n",
       "              GSM856495_oocytes-BDF1-1  GSM856516_2cellNoPB2-BDF1-X1-3  \\\n",
       "index                                                                    \n",
       "chr1_3010894                       NaN                             NaN   \n",
       "chr1_3010922                       NaN                             NaN   \n",
       "chr1_3089715                  0.377778                        0.527778   \n",
       "chr1_3089865                  0.563636                        0.466667   \n",
       "chr1_3102314                       NaN                             NaN   \n",
       "\n",
       "              GSM856508_2cell-BDF1-BDF1-1  GSM856530_8cell-BDF1-BDF1-3  \\\n",
       "index                                                                    \n",
       "chr1_3010894                          NaN                     0.562500   \n",
       "chr1_3010922                          NaN                     0.818182   \n",
       "chr1_3089715                          NaN                     0.389831   \n",
       "chr1_3089865                          NaN                     0.333333   \n",
       "chr1_3102314                          NaN                          NaN   \n",
       "\n",
       "              GSM856550_E75-BDF1-BDF1-1  GSM870007_sperm-X1-3  ...  \\\n",
       "index                                                          ...   \n",
       "chr1_3010894                   0.870968                   NaN  ...   \n",
       "chr1_3010922                        NaN                   NaN  ...   \n",
       "chr1_3089715                   0.846154                   NaN  ...   \n",
       "chr1_3089865                   0.813953              0.823529  ...   \n",
       "chr1_3102314                        NaN                   NaN  ...   \n",
       "\n",
       "              GSM856526_4cellNoPB2-BDF1-X1-2  GSM856502_zygote-BDF1-BDF1-2  \\\n",
       "index                                                                        \n",
       "chr1_3010894                             NaN                           NaN   \n",
       "chr1_3010922                             NaN                           NaN   \n",
       "chr1_3089715                        0.400000                           NaN   \n",
       "chr1_3089865                        0.424242                           NaN   \n",
       "chr1_3102314                             NaN                           NaN   \n",
       "\n",
       "              GSM856507_zygoteNoPB-BDF1-X1-3  GSM856535_8cellNoPB-BDF1-X1-2  \\\n",
       "index                                                                         \n",
       "chr1_3010894                             NaN                            NaN   \n",
       "chr1_3010922                             NaN                            NaN   \n",
       "chr1_3089715                        0.426230                       0.324324   \n",
       "chr1_3089865                        0.509434                       0.446809   \n",
       "chr1_3102314                             NaN                            NaN   \n",
       "\n",
       "              GSM856525_4cellNoPB2-BDF1-X1-1  GSM870002_zygoteNoPB2-BDF1-X1-5  \\\n",
       "index                                                                           \n",
       "chr1_3010894                             NaN                              NaN   \n",
       "chr1_3010922                             NaN                              NaN   \n",
       "chr1_3089715                             NaN                         0.409091   \n",
       "chr1_3089865                             NaN                         0.409836   \n",
       "chr1_3102314                             NaN                              NaN   \n",
       "\n",
       "              GSM856509_2cell-BDF1-X1-2  GSM856515_2cellNoPB2-BDF1-X1-2  \\\n",
       "index                                                                     \n",
       "chr1_3010894                        NaN                             NaN   \n",
       "chr1_3010922                        NaN                             NaN   \n",
       "chr1_3089715                        NaN                          0.5625   \n",
       "chr1_3089865                        NaN                             NaN   \n",
       "chr1_3102314                        NaN                             NaN   \n",
       "\n",
       "              GSM856556_Brain-2  GSM856505_zygoteNoPB-BDF1-X1-1  \n",
       "index                                                            \n",
       "chr1_3010894           0.827586                             NaN  \n",
       "chr1_3010922           0.917647                             NaN  \n",
       "chr1_3089715           0.936170                        0.435897  \n",
       "chr1_3089865                NaN                        0.525000  \n",
       "chr1_3102314                NaN                             NaN  \n",
       "\n",
       "[5 rows x 76 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_path = Path.joinpath(cwd, project_dir, \"{}.csv\".format(project_id))\n",
    "print(data_path)\n",
    "\n",
    "# df_coverage = mtools.get_methylation_df(data_path, typ='coverage')\n",
    "df_percent = mtools.get_methylation_df(data_path, typ='percents')\n",
    "print(df_percent.shape)\n",
    "df_percent.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b46a93a5-043c-4dbf-b18d-fdeb09e2763a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b8efd5bd-478b-4ca3-b778-1334de63d388",
   "metadata": {},
   "outputs": [],
   "source": [
    "metainfo_old_path = 'GSE34864_mousePMD100bpMe.txt'\n",
    "metainfo_path = 'GSE34864_Metainfo.csv'\n",
    "gl_metainfo_old_path = Path.joinpath(project_dir, metainfo_old_path)\n",
    "gl_metainfo_path = Path.joinpath(project_dir, metainfo_path)\n",
    "data_path = Path.joinpath(cwd, project_dir, \"{}.csv\".format(project_id))\n",
    "gl_project_dir = Path.joinpath(cwd, project_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "86b1725d-ffed-4183-8b07-24cd71889b81",
   "metadata": {},
   "outputs": [],
   "source": [
    "stage2age = {'ICM': 5.0,\n",
    "             'zygote': 0.5,\n",
    "             'E75':7.5, '8cellNoPB2': 2.5, 'oocytes': 0.0, '2cellNoPB2': 1.5,\n",
    "       '2cell': 1.5, '8cell': 2.5, 'sperm': 0.0, 'zygoteNoPB': 0.5, 'E65': 6.5, '8cellNoPB': 2.5,\n",
    "       'sperm2': 0.0, '4cellNoPB': 2.0, '2cellNoPB': 1.5, 'zygoteNoPB2': 0.5, '4cell': 2.0,\n",
    "       'Liver': 7*30., 'Heart': 7*30., 'Brain': 7*30., '4cellNoPB2': 2.0\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "7a30ad14-0b98-46a1-9ad4-f202c07114d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GSM</th>\n",
       "      <th>SampleID</th>\n",
       "      <th>SampleInfo</th>\n",
       "      <th>Age</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FullID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>GSM856541_ICM-BDF1-AJ-3a</th>\n",
       "      <td>GSM856541</td>\n",
       "      <td>ICM</td>\n",
       "      <td>BDF1-AJ-3a</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSM856503_zygote-BDF1-X1-3</th>\n",
       "      <td>GSM856503</td>\n",
       "      <td>zygote</td>\n",
       "      <td>BDF1-X1-3</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSM856551_E75-BDF1-BDF1-2</th>\n",
       "      <td>GSM856551</td>\n",
       "      <td>E75</td>\n",
       "      <td>BDF1-BDF1-2</td>\n",
       "      <td>7.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSM856537_8cellNoPB2-BDF1-X1-2</th>\n",
       "      <td>GSM856537</td>\n",
       "      <td>8cellNoPB2</td>\n",
       "      <td>BDF1-X1-2</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSM856495_oocytes-BDF1-1</th>\n",
       "      <td>GSM856495</td>\n",
       "      <td>oocytes</td>\n",
       "      <td>BDF1-1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      GSM    SampleID   SampleInfo  Age\n",
       "FullID                                                                 \n",
       "GSM856541_ICM-BDF1-AJ-3a        GSM856541         ICM   BDF1-AJ-3a  5.0\n",
       "GSM856503_zygote-BDF1-X1-3      GSM856503      zygote    BDF1-X1-3  0.5\n",
       "GSM856551_E75-BDF1-BDF1-2       GSM856551         E75  BDF1-BDF1-2  7.5\n",
       "GSM856537_8cellNoPB2-BDF1-X1-2  GSM856537  8cellNoPB2    BDF1-X1-2  2.5\n",
       "GSM856495_oocytes-BDF1-1        GSM856495     oocytes       BDF1-1  0.0"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df_meta = pd.read_csv(data_path, skiprows=0, nrows=0, index_col=0, sep='\\t')\n",
    "df_meta = pd.DataFrame(df_percent.columns, columns=['FullID'])\n",
    "df_meta['GSM'] = df_meta.iloc[:,0].apply(lambda x: x.split('_')[0])\n",
    "df_meta['SampleID'] = df_meta.iloc[:,0].apply(lambda x: x.split('_')[1].split('-')[0])\n",
    "df_meta['SampleInfo'] = df_meta.iloc[:,0].apply(lambda x: \"-\".join(x.split('_')[1].split('-')[1:]))\n",
    "df_meta['Age'] = df_meta['SampleID'].apply(lambda x: stage2age[x])\n",
    "df_meta.set_index('FullID', inplace=True)\n",
    "\n",
    "to_category = ['GSM', 'SampleID', 'SampleInfo']\n",
    "df_meta[to_category] = df_meta[to_category].astype(\"category\")\n",
    "\n",
    "df_meta.to_csv(gl_metainfo_path, sep='\\t')\n",
    "df_meta.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "162a9a3c-38ba-4cf8-9840-3e5308451347",
   "metadata": {},
   "outputs": [],
   "source": [
    "ad = sc.AnnData(df_percent)\n",
    "ad.var = pd.concat((ad.var, df_meta.loc[ad.var.index,:]), axis=1)\n",
    "ad.X = ad.X.astype(np.float16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "d6299496-5168-4857-8f7a-0ee047b98e9e",
   "metadata": {},
   "outputs": [],
   "source": [
    "ad.write_h5ad(Path.joinpath(project_dir, \"{}.h5ad\".format(project_id)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5e8526d-ee8e-4764-8010-c35b87d765b1",
   "metadata": {},
   "source": [
    "# Bohan parabiosis liver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "e812ada0-743a-4511-896e-fdf266d1610e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import pwd\n",
    "from pathlib import Path\n",
    "from tqdm import tqdm, trange\n",
    "from tqdm.contrib.concurrent import process_map\n",
    "import meth.tools as mtools\n",
    "import numpy as np\n",
    "\n",
    "cwd = Path.cwd()\n",
    "project_dir = Path('data/Parabiosis_liver/')\n",
    "project_id = 'Parabiosis_liver'\n",
    "project_samples_dir = 'Parabiosis_liver_RAW'\n",
    "path_samples = Path.joinpath(project_dir, project_samples_dir)\n",
    "gl_path_samples = Path.joinpath(cwd, path_samples)\n",
    "gl_project_dir = Path.joinpath(cwd, project_dir)\n",
    "all_samples = os.listdir(gl_path_samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "1a3f845b-6692-4bce-8d1e-2c7353ae2c75",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/tarkhov/Dropbox/Harvard/Science/Stochastic aging/data/Parabiosis_liver/Parabiosis_liver_coverage_0_binned_by_10.csv\n",
      "(4178255, 46)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BZ64</th>\n",
       "      <th>BZ25</th>\n",
       "      <th>BZ43</th>\n",
       "      <th>BZ81</th>\n",
       "      <th>BZ11</th>\n",
       "      <th>BZ42</th>\n",
       "      <th>BZ16</th>\n",
       "      <th>BZ41</th>\n",
       "      <th>BZ82</th>\n",
       "      <th>BZ66</th>\n",
       "      <th>...</th>\n",
       "      <th>BZ63</th>\n",
       "      <th>BZ47</th>\n",
       "      <th>BZ72</th>\n",
       "      <th>BZ55</th>\n",
       "      <th>BZ45</th>\n",
       "      <th>BZ75</th>\n",
       "      <th>BZ26</th>\n",
       "      <th>BZ34</th>\n",
       "      <th>BZ35</th>\n",
       "      <th>BZ44</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>chr10_100012200</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.0000</td>\n",
       "      <td>87.878788</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.00</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.0</td>\n",
       "      <td>98.412698</td>\n",
       "      <td>96.666667</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr10_100012210</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>5.882353</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.166667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.894737</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.407407</td>\n",
       "      <td>7.142857</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.263158</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>31.250000</td>\n",
       "      <td>11.627907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr10_100012220</th>\n",
       "      <td>96.296296</td>\n",
       "      <td>98.4375</td>\n",
       "      <td>98.571429</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>97.101449</td>\n",
       "      <td>98.666667</td>\n",
       "      <td>96.153846</td>\n",
       "      <td>95.312500</td>\n",
       "      <td>100.00</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>96.470588</td>\n",
       "      <td>90.0</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>93.333333</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>93.333333</td>\n",
       "      <td>97.222222</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>97.701149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr10_100012230</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>90.6250</td>\n",
       "      <td>91.428571</td>\n",
       "      <td>93.181818</td>\n",
       "      <td>86.956522</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>95.312500</td>\n",
       "      <td>100.00</td>\n",
       "      <td>91.489362</td>\n",
       "      <td>...</td>\n",
       "      <td>85.882353</td>\n",
       "      <td>95.0</td>\n",
       "      <td>98.913043</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>87.272727</td>\n",
       "      <td>91.891892</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>94.252874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr10_100012250</th>\n",
       "      <td>97.530864</td>\n",
       "      <td>95.3125</td>\n",
       "      <td>82.857143</td>\n",
       "      <td>93.181818</td>\n",
       "      <td>89.855072</td>\n",
       "      <td>96.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>92.187500</td>\n",
       "      <td>93.75</td>\n",
       "      <td>80.851064</td>\n",
       "      <td>...</td>\n",
       "      <td>89.411765</td>\n",
       "      <td>80.0</td>\n",
       "      <td>92.391304</td>\n",
       "      <td>95.555556</td>\n",
       "      <td>92.727273</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>93.333333</td>\n",
       "      <td>63.888889</td>\n",
       "      <td>94.594595</td>\n",
       "      <td>88.505747</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 46 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       BZ64      BZ25       BZ43        BZ81        BZ11  \\\n",
       "index                                                                      \n",
       "chr10_100012200  100.000000  100.0000  87.878788  100.000000  100.000000   \n",
       "chr10_100012210    0.000000    0.0000   5.882353    0.000000    4.000000   \n",
       "chr10_100012220   96.296296   98.4375  98.571429  100.000000   97.101449   \n",
       "chr10_100012230  100.000000   90.6250  91.428571   93.181818   86.956522   \n",
       "chr10_100012250   97.530864   95.3125  82.857143   93.181818   89.855072   \n",
       "\n",
       "                       BZ42        BZ16        BZ41    BZ82        BZ66  ...  \\\n",
       "index                                                                    ...   \n",
       "chr10_100012200  100.000000  100.000000  100.000000  100.00  100.000000  ...   \n",
       "chr10_100012210    4.166667    0.000000    7.894737    0.00    0.000000  ...   \n",
       "chr10_100012220   98.666667   96.153846   95.312500  100.00  100.000000  ...   \n",
       "chr10_100012230  100.000000  100.000000   95.312500  100.00   91.489362  ...   \n",
       "chr10_100012250   96.000000  100.000000   92.187500   93.75   80.851064  ...   \n",
       "\n",
       "                       BZ63   BZ47        BZ72        BZ55        BZ45  \\\n",
       "index                                                                    \n",
       "chr10_100012200  100.000000  100.0   98.412698   96.666667  100.000000   \n",
       "chr10_100012210   10.000000    0.0    7.407407    7.142857    5.000000   \n",
       "chr10_100012220   96.470588   90.0  100.000000   93.333333  100.000000   \n",
       "chr10_100012230   85.882353   95.0   98.913043  100.000000   87.272727   \n",
       "chr10_100012250   89.411765   80.0   92.391304   95.555556   92.727273   \n",
       "\n",
       "                       BZ75        BZ26        BZ34        BZ35        BZ44  \n",
       "index                                                                        \n",
       "chr10_100012200  100.000000  100.000000  100.000000  100.000000  100.000000  \n",
       "chr10_100012210    0.000000    5.263158    0.000000   31.250000   11.627907  \n",
       "chr10_100012220  100.000000   93.333333   97.222222  100.000000   97.701149  \n",
       "chr10_100012230   91.891892  100.000000  100.000000  100.000000   94.252874  \n",
       "chr10_100012250  100.000000   93.333333   63.888889   94.594595   88.505747  \n",
       "\n",
       "[5 rows x 46 columns]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_path = Path.joinpath(cwd, project_dir, \"{}_coverage_0_binned_by_10.csv\".format(project_id))\n",
    "print(data_path)\n",
    "df_percent = mtools.get_methylation_df(data_path, typ='percents')\n",
    "print(df_percent.shape)\n",
    "df_percent.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "f0aab9aa-253c-48db-a449-3fbd1b106aac",
   "metadata": {},
   "outputs": [],
   "source": [
    "group_description = {'BZ1': \"all young iso attached to the young\", \n",
    "                     'BZ2': \"young iso attached to the young with detachment\", \n",
    "                     'BZ3': \"old iso attached to the old\",\n",
    "                     'BZ4': \"old iso attached to the old with detachment\", \n",
    "                     'BZ5': \"old het attached to the young\", \n",
    "                     'BZ6': \"old het attached to the young with detachment\",\n",
    "                     'BZ7': \"young het attached to the old\",\n",
    "                     'BZ8': \"young het attached to the old with detachment\"}\n",
    "\n",
    "metainfo_path = 'Liver_Parabiosis_Metainfo.csv'\n",
    "gl_metainfo_path = Path.joinpath(project_dir, metainfo_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "bac441ef-25c9-4f9b-a53a-adfd3100d8d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tissue</th>\n",
       "      <th>GroupID</th>\n",
       "      <th>MouseID</th>\n",
       "      <th>GroupDescription</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SampleID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>BZ64</th>\n",
       "      <td>Liver</td>\n",
       "      <td>BZ6</td>\n",
       "      <td>4</td>\n",
       "      <td>old het attached to the young with detachment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BZ25</th>\n",
       "      <td>Liver</td>\n",
       "      <td>BZ2</td>\n",
       "      <td>5</td>\n",
       "      <td>young iso attached to the young with detachment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BZ43</th>\n",
       "      <td>Liver</td>\n",
       "      <td>BZ4</td>\n",
       "      <td>3</td>\n",
       "      <td>old iso attached to the old with detachment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BZ81</th>\n",
       "      <td>Liver</td>\n",
       "      <td>BZ8</td>\n",
       "      <td>1</td>\n",
       "      <td>young het attached to the old with detachment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BZ11</th>\n",
       "      <td>Liver</td>\n",
       "      <td>BZ1</td>\n",
       "      <td>1</td>\n",
       "      <td>all young iso attached to the young</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Tissue GroupID MouseID  \\\n",
       "SampleID                          \n",
       "BZ64      Liver     BZ6       4   \n",
       "BZ25      Liver     BZ2       5   \n",
       "BZ43      Liver     BZ4       3   \n",
       "BZ81      Liver     BZ8       1   \n",
       "BZ11      Liver     BZ1       1   \n",
       "\n",
       "                                         GroupDescription  \n",
       "SampleID                                                   \n",
       "BZ64        old het attached to the young with detachment  \n",
       "BZ25      young iso attached to the young with detachment  \n",
       "BZ43          old iso attached to the old with detachment  \n",
       "BZ81        young het attached to the old with detachment  \n",
       "BZ11                  all young iso attached to the young  "
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_meta = pd.DataFrame(df_percent.columns, columns=['SampleID'])\n",
    "df_meta['Tissue'] = 'Liver'\n",
    "df_meta['GroupID'] = df_meta['SampleID'].apply(lambda x: x[:3])\n",
    "df_meta['MouseID'] = df_meta['SampleID'].apply(lambda x: x[3:])\n",
    "df_meta['GroupDescription'] = df_meta['GroupID'].apply(lambda x: group_description[x])\n",
    "df_meta.set_index('SampleID', inplace=True)\n",
    "to_category = ['Tissue', 'GroupID', 'MouseID', \"GroupDescription\"]\n",
    "df_meta[to_category] = df_meta[to_category].astype(\"category\")\n",
    "df_meta.to_csv(gl_metainfo_path, sep='\\t')\n",
    "df_meta.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "ec3879ac-64a2-4aa3-b328-85c3f9ce3e14",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tissue</th>\n",
       "      <th>GroupID</th>\n",
       "      <th>MouseID</th>\n",
       "      <th>GroupDescription</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SampleID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>BZ64</th>\n",
       "      <td>Liver</td>\n",
       "      <td>BZ6</td>\n",
       "      <td>4</td>\n",
       "      <td>old het attached to the young with detachment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BZ25</th>\n",
       "      <td>Liver</td>\n",
       "      <td>BZ2</td>\n",
       "      <td>5</td>\n",
       "      <td>young iso attached to the young with detachment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BZ43</th>\n",
       "      <td>Liver</td>\n",
       "      <td>BZ4</td>\n",
       "      <td>3</td>\n",
       "      <td>old iso attached to the old with detachment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BZ81</th>\n",
       "      <td>Liver</td>\n",
       "      <td>BZ8</td>\n",
       "      <td>1</td>\n",
       "      <td>young het attached to the old with detachment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BZ11</th>\n",
       "      <td>Liver</td>\n",
       "      <td>BZ1</td>\n",
       "      <td>1</td>\n",
       "      <td>all young iso attached to the young</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Tissue GroupID  MouseID  \\\n",
       "SampleID                           \n",
       "BZ64      Liver     BZ6        4   \n",
       "BZ25      Liver     BZ2        5   \n",
       "BZ43      Liver     BZ4        3   \n",
       "BZ81      Liver     BZ8        1   \n",
       "BZ11      Liver     BZ1        1   \n",
       "\n",
       "                                         GroupDescription  \n",
       "SampleID                                                   \n",
       "BZ64        old het attached to the young with detachment  \n",
       "BZ25      young iso attached to the young with detachment  \n",
       "BZ43          old iso attached to the old with detachment  \n",
       "BZ81        young het attached to the old with detachment  \n",
       "BZ11                  all young iso attached to the young  "
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_meta = pd.read_csv(gl_metainfo_path, sep='\\t', index_col=0)\n",
    "df_meta.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "6ed69619-e1f3-4400-94bb-ae017fedc8ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "ad = sc.AnnData(df_percent)\n",
    "ad.var = pd.concat((ad.var, df_meta.loc[ad.var.index,:]), axis=1)\n",
    "ad.X = ad.X.astype(np.float16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "4665187a-9c67-44f5-8e5b-01fdf3fca933",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'Tissue' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'GroupID' as categorical\n",
      "/Users/tarkhov/opt/anaconda3/envs/py310/lib/python3.10/site-packages/anndata/_core/anndata.py:1228: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Reordering categories will always return a new Categorical object.\n",
      "  c.reorder_categories(natsorted(c.categories), inplace=True)\n",
      "... storing 'GroupDescription' as categorical\n"
     ]
    }
   ],
   "source": [
    "ad.write_h5ad(Path.joinpath(project_dir, \"{}.h5ad\".format(project_id)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d70f8cc-b05f-4c3b-b2b0-d6de1d59b6db",
   "metadata": {},
   "source": [
    "# Bohan parabiosis blood"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "a064a0b4-85f3-4aee-9265-df78ad289136",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import pwd\n",
    "from pathlib import Path\n",
    "from tqdm import tqdm, trange\n",
    "from tqdm.contrib.concurrent import process_map\n",
    "import meth.tools as mtools\n",
    "import numpy as np\n",
    "\n",
    "cwd = Path.cwd()\n",
    "project_dir = Path('data/Parabiosis_blood/')\n",
    "project_id = 'Parabiosis_blood'\n",
    "project_samples_dir = 'Parabiosis_blood_RAW'\n",
    "path_samples = Path.joinpath(project_dir, project_samples_dir)\n",
    "gl_path_samples = Path.joinpath(cwd, path_samples)\n",
    "gl_project_dir = Path.joinpath(cwd, project_dir)\n",
    "all_samples = os.listdir(gl_path_samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "e347c368-336b-40e8-81fe-b71282f0fa59",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/tarkhov/Dropbox/Harvard/Science/Stochastic aging/data/Parabiosis_blood/Parabiosis_blood_coverage_0_binned_by_10.csv\n",
      "(6394251, 32)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BldBZ35</th>\n",
       "      <th>BldBZ14</th>\n",
       "      <th>BldBZ26</th>\n",
       "      <th>BldBZ55</th>\n",
       "      <th>BldBZ41</th>\n",
       "      <th>BldBZ34</th>\n",
       "      <th>BldBZ31</th>\n",
       "      <th>BldBZ15</th>\n",
       "      <th>BldBZ61</th>\n",
       "      <th>BldBZ43</th>\n",
       "      <th>...</th>\n",
       "      <th>BldBZ51</th>\n",
       "      <th>BldBZ45</th>\n",
       "      <th>BldBZ33</th>\n",
       "      <th>BldBZ12</th>\n",
       "      <th>BldBZ11</th>\n",
       "      <th>BldBZ32</th>\n",
       "      <th>BldBZ62</th>\n",
       "      <th>BldBZ44</th>\n",
       "      <th>BldBZ13</th>\n",
       "      <th>BldBZ52</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>chr10_100012200</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>95.454545</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>90.625</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>96.428571</td>\n",
       "      <td>100.000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>94.339623</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>87.878788</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>95.454545</td>\n",
       "      <td>97.297297</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>85.937500</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr10_100012210</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>12.500000</td>\n",
       "      <td>7.142857</td>\n",
       "      <td>0.000</td>\n",
       "      <td>15.625000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>13.333333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>10.256410</td>\n",
       "      <td>7.692308</td>\n",
       "      <td>10.526316</td>\n",
       "      <td>18.750000</td>\n",
       "      <td>3.703704</td>\n",
       "      <td>2.857143</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>14.705882</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr10_100012220</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>97.826087</td>\n",
       "      <td>94.382022</td>\n",
       "      <td>97.500</td>\n",
       "      <td>96.296296</td>\n",
       "      <td>96.078431</td>\n",
       "      <td>96.875</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>94.117647</td>\n",
       "      <td>90.476190</td>\n",
       "      <td>...</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>98.913043</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>97.183099</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>95.000000</td>\n",
       "      <td>99.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr10_100012230</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>97.752809</td>\n",
       "      <td>100.000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>98.039216</td>\n",
       "      <td>100.000</td>\n",
       "      <td>98.717949</td>\n",
       "      <td>98.529412</td>\n",
       "      <td>92.063492</td>\n",
       "      <td>...</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>98.591549</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>95.000000</td>\n",
       "      <td>97.000000</td>\n",
       "      <td>94.594595</td>\n",
       "      <td>98.684211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr10_100012250</th>\n",
       "      <td>97.222222</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>96.629213</td>\n",
       "      <td>97.500</td>\n",
       "      <td>83.333333</td>\n",
       "      <td>90.196078</td>\n",
       "      <td>90.625</td>\n",
       "      <td>98.717949</td>\n",
       "      <td>95.588235</td>\n",
       "      <td>84.126984</td>\n",
       "      <td>...</td>\n",
       "      <td>95.238095</td>\n",
       "      <td>95.652174</td>\n",
       "      <td>75.000000</td>\n",
       "      <td>96.078431</td>\n",
       "      <td>95.774648</td>\n",
       "      <td>95.833333</td>\n",
       "      <td>86.666667</td>\n",
       "      <td>93.000000</td>\n",
       "      <td>98.648649</td>\n",
       "      <td>92.105263</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    BldBZ35     BldBZ14     BldBZ26  BldBZ55     BldBZ41  \\\n",
       "index                                                                      \n",
       "chr10_100012200  100.000000   95.454545  100.000000   90.625  100.000000   \n",
       "chr10_100012210    0.000000   12.500000    7.142857    0.000   15.625000   \n",
       "chr10_100012220  100.000000   97.826087   94.382022   97.500   96.296296   \n",
       "chr10_100012230  100.000000  100.000000   97.752809  100.000  100.000000   \n",
       "chr10_100012250   97.222222  100.000000   96.629213   97.500   83.333333   \n",
       "\n",
       "                   BldBZ34  BldBZ31     BldBZ15     BldBZ61     BldBZ43  ...  \\\n",
       "index                                                                    ...   \n",
       "chr10_100012200  96.428571  100.000  100.000000  100.000000  100.000000  ...   \n",
       "chr10_100012210   0.000000    0.000   13.333333    0.000000    0.000000  ...   \n",
       "chr10_100012220  96.078431   96.875  100.000000   94.117647   90.476190  ...   \n",
       "chr10_100012230  98.039216  100.000   98.717949   98.529412   92.063492  ...   \n",
       "chr10_100012250  90.196078   90.625   98.717949   95.588235   84.126984  ...   \n",
       "\n",
       "                    BldBZ51     BldBZ45     BldBZ33     BldBZ12    BldBZ11  \\\n",
       "index                                                                        \n",
       "chr10_100012200   94.339623  100.000000   87.878788  100.000000  95.454545   \n",
       "chr10_100012210   10.256410    7.692308   10.526316   18.750000   3.703704   \n",
       "chr10_100012220  100.000000   98.913043  100.000000  100.000000  97.183099   \n",
       "chr10_100012230  100.000000  100.000000  100.000000  100.000000  98.591549   \n",
       "chr10_100012250   95.238095   95.652174   75.000000   96.078431  95.774648   \n",
       "\n",
       "                    BldBZ32     BldBZ62    BldBZ44     BldBZ13     BldBZ52  \n",
       "index                                                                       \n",
       "chr10_100012200   97.297297  100.000000  85.937500  100.000000  100.000000  \n",
       "chr10_100012210    2.857143    0.000000  14.705882    5.000000    0.000000  \n",
       "chr10_100012220  100.000000   95.000000  99.000000  100.000000  100.000000  \n",
       "chr10_100012230  100.000000   95.000000  97.000000   94.594595   98.684211  \n",
       "chr10_100012250   95.833333   86.666667  93.000000   98.648649   92.105263  \n",
       "\n",
       "[5 rows x 32 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_path = Path.joinpath(cwd, project_dir, \"{}_coverage_0_binned_by_10.csv\".format(project_id))\n",
    "print(data_path)\n",
    "df_percent = mtools.get_methylation_df(data_path, typ='percents')\n",
    "print(df_percent.shape)\n",
    "df_percent.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "00818a31-fbd3-4e0b-a00c-841f9b0317a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "group_description = {'BZ1': \"all young iso attached to the young\", \n",
    "                     'BZ2': \"young iso attached to the young with detachment\", \n",
    "                     'BZ3': \"old iso attached to the old\",\n",
    "                     'BZ4': \"old iso attached to the old with detachment\", \n",
    "                     'BZ5': \"old het attached to the young\", \n",
    "                     'BZ6': \"old het attached to the young with detachment\",\n",
    "                     'BZ7': \"young het attached to the old\",\n",
    "                     'BZ8': \"young het attached to the old with detachment\"}\n",
    "\n",
    "metainfo_path = 'Blood_Parabiosis_Metainfo.csv'\n",
    "gl_metainfo_path = Path.joinpath(project_dir, metainfo_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "954ea19e-e4a3-4b6b-a9de-d67fc90966ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tissue</th>\n",
       "      <th>GroupID</th>\n",
       "      <th>MouseID</th>\n",
       "      <th>GroupDescription</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SampleID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>BldBZ35</th>\n",
       "      <td>Blood</td>\n",
       "      <td>BZ3</td>\n",
       "      <td>5</td>\n",
       "      <td>old iso attached to the old</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BldBZ14</th>\n",
       "      <td>Blood</td>\n",
       "      <td>BZ1</td>\n",
       "      <td>4</td>\n",
       "      <td>all young iso attached to the young</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BldBZ26</th>\n",
       "      <td>Blood</td>\n",
       "      <td>BZ2</td>\n",
       "      <td>6</td>\n",
       "      <td>young iso attached to the young with detachment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BldBZ55</th>\n",
       "      <td>Blood</td>\n",
       "      <td>BZ5</td>\n",
       "      <td>5</td>\n",
       "      <td>old het attached to the young</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BldBZ41</th>\n",
       "      <td>Blood</td>\n",
       "      <td>BZ4</td>\n",
       "      <td>1</td>\n",
       "      <td>old iso attached to the old with detachment</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Tissue GroupID MouseID  \\\n",
       "SampleID                          \n",
       "BldBZ35   Blood     BZ3       5   \n",
       "BldBZ14   Blood     BZ1       4   \n",
       "BldBZ26   Blood     BZ2       6   \n",
       "BldBZ55   Blood     BZ5       5   \n",
       "BldBZ41   Blood     BZ4       1   \n",
       "\n",
       "                                         GroupDescription  \n",
       "SampleID                                                   \n",
       "BldBZ35                       old iso attached to the old  \n",
       "BldBZ14               all young iso attached to the young  \n",
       "BldBZ26   young iso attached to the young with detachment  \n",
       "BldBZ55                     old het attached to the young  \n",
       "BldBZ41       old iso attached to the old with detachment  "
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_meta = pd.DataFrame(df_percent.columns, columns=['SampleID'])\n",
    "df_meta['Tissue'] = 'Blood'\n",
    "df_meta['GroupID'] = df_meta['SampleID'].apply(lambda x: x[3:6])\n",
    "df_meta['MouseID'] = df_meta['SampleID'].apply(lambda x: x[6:])\n",
    "df_meta['GroupDescription'] = df_meta['GroupID'].apply(lambda x: group_description[x])\n",
    "df_meta.set_index('SampleID', inplace=True)\n",
    "to_category = ['Tissue', 'GroupID', 'MouseID', \"GroupDescription\"]\n",
    "df_meta[to_category] = df_meta[to_category].astype(\"category\")\n",
    "\n",
    "df_meta.to_csv(gl_metainfo_path, sep='\\t')\n",
    "df_meta.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "ac24a1e2-2727-4587-83f3-442a34be18b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "ad = sc.AnnData(df_percent)\n",
    "ad.var = pd.concat((ad.var, df_meta.loc[ad.var.index,:]), axis=1)\n",
    "ad.X = ad.X.astype(np.float16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "a561ae27-8ad9-419a-a362-13478af615c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "ad.write_h5ad(Path.joinpath(project_dir, \"{}.h5ad\".format(project_id)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d88da550-d3b2-4ccd-8d2f-e2093c8e6e37",
   "metadata": {},
   "source": [
    "# Analysing the data\n",
    "The idea is to extract those sites that are decreasing or increasing their methylation level (exactly from 0 to 1), and estimate the average rate of that. It must be the same in all species (the relative one!). That would mean that the temperature is the limiting factor, whereas the methylation clocks are just measuring the effects of stochastic environmental factors on the organism. The moment the methylation reaches 0.5 means that the organism is dead. Hence, stochasticity is the sole reason limiting the lifespan."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48e0b7ae-2b9a-4213-b72f-c2bfe071f3b8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python3.10 (biology)",
   "language": "python",
   "name": "py310"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}