matrix_conversion_explore.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import feather"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "matrix = feather.read_dataframe('./fib.fafb.crossnblast.twigs5.mean.compress.feather')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>1001453586</th>\n",
       "      <th>1001971876</th>\n",
       "      <th>1002360103</th>\n",
       "      <th>1002507131</th>\n",
       "      <th>1002507159</th>\n",
       "      <th>1002507170</th>\n",
       "      <th>1002627843</th>\n",
       "      <th>1002658094</th>\n",
       "      <th>1002848124</th>\n",
       "      <th>...</th>\n",
       "      <th>987424258</th>\n",
       "      <th>987759904</th>\n",
       "      <th>987877232</th>\n",
       "      <th>987997896</th>\n",
       "      <th>988269593</th>\n",
       "      <th>988567837</th>\n",
       "      <th>988632865</th>\n",
       "      <th>988852391</th>\n",
       "      <th>988909130</th>\n",
       "      <th>989228019</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>16</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.47</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>27</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.485</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.407</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.361</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>430</td>\n",
       "      <td>-0.394</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.282</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.457</td>\n",
       "      <td>-0.492</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.498</td>\n",
       "      <td>-0.479</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.429</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.44</td>\n",
       "      <td>-0.438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>435</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.464</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.324</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>717</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.335</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.393</td>\n",
       "      <td>-0.373</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.489</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.371</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27412 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  index  1001453586  1001971876  1002360103  1002507131  1002507159  \\\n",
       "0    16      -0.500      -0.500      -0.500        -0.5        -0.5   \n",
       "1    27      -0.500      -0.485      -0.500        -0.5        -0.5   \n",
       "2   430      -0.394      -0.500      -0.282        -0.5        -0.5   \n",
       "3   435      -0.500      -0.500      -0.500        -0.5        -0.5   \n",
       "4   717      -0.500      -0.335      -0.500        -0.5        -0.5   \n",
       "\n",
       "   1002507170  1002627843  1002658094  1002848124  ...  987424258  987759904  \\\n",
       "0       -0.47      -0.500      -0.500      -0.500  ...       -0.5     -0.500   \n",
       "1       -0.50      -0.500      -0.500      -0.500  ...       -0.5     -0.500   \n",
       "2       -0.50      -0.500      -0.457      -0.492  ...       -0.5     -0.498   \n",
       "3       -0.50      -0.500      -0.500      -0.464  ...       -0.5     -0.500   \n",
       "4       -0.50      -0.393      -0.373      -0.500  ...       -0.5     -0.500   \n",
       "\n",
       "   987877232  987997896  988269593  988567837  988632865  988852391  \\\n",
       "0     -0.500       -0.5     -0.500     -0.500     -0.500       -0.5   \n",
       "1     -0.500       -0.5     -0.407     -0.500     -0.361       -0.5   \n",
       "2     -0.479       -0.5     -0.500     -0.429     -0.500       -0.5   \n",
       "3     -0.500       -0.5     -0.500     -0.500     -0.324       -0.5   \n",
       "4     -0.500       -0.5     -0.489     -0.500     -0.371       -0.5   \n",
       "\n",
       "   988909130  989228019  \n",
       "0      -0.50     -0.500  \n",
       "1      -0.50     -0.500  \n",
       "2      -0.44     -0.438  \n",
       "3      -0.50     -0.500  \n",
       "4      -0.50     -0.500  \n",
       "\n",
       "[5 rows x 27412 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matrix[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['index', '1001453586', '1001971876', '1002360103', '1002507131',\n",
       "       '1002507159', '1002507170', '1002627843', '1002658094', '1002848124'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check columns (expect hemibrain BodyIds)\n",
    "matrix.columns[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      16\n",
       "1      27\n",
       "2     430\n",
       "3     435\n",
       "4     717\n",
       "5     734\n",
       "6     949\n",
       "7    1165\n",
       "8    1992\n",
       "9    2076\n",
       "Name: index, dtype: object"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# First column is the index we need (FAFB skids)\n",
    "matrix['index'][0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1001453586</th>\n",
       "      <th>1001971876</th>\n",
       "      <th>1002360103</th>\n",
       "      <th>1002507131</th>\n",
       "      <th>1002507159</th>\n",
       "      <th>1002507170</th>\n",
       "      <th>1002627843</th>\n",
       "      <th>1002658094</th>\n",
       "      <th>1002848124</th>\n",
       "      <th>1002848661</th>\n",
       "      <th>...</th>\n",
       "      <th>987424258</th>\n",
       "      <th>987759904</th>\n",
       "      <th>987877232</th>\n",
       "      <th>987997896</th>\n",
       "      <th>988269593</th>\n",
       "      <th>988567837</th>\n",
       "      <th>988632865</th>\n",
       "      <th>988852391</th>\n",
       "      <th>988909130</th>\n",
       "      <th>989228019</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.47</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.485</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.407</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.361</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>430</th>\n",
       "      <td>-0.394</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.282</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.457</td>\n",
       "      <td>-0.492</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.498</td>\n",
       "      <td>-0.479</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.429</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.44</td>\n",
       "      <td>-0.438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>435</th>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.464</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.324</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>717</th>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.335</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.393</td>\n",
       "      <td>-0.373</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.489</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.371</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27411 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       1001453586  1001971876  1002360103  1002507131  1002507159  1002507170  \\\n",
       "index                                                                           \n",
       "16         -0.500      -0.500      -0.500        -0.5        -0.5       -0.47   \n",
       "27         -0.500      -0.485      -0.500        -0.5        -0.5       -0.50   \n",
       "430        -0.394      -0.500      -0.282        -0.5        -0.5       -0.50   \n",
       "435        -0.500      -0.500      -0.500        -0.5        -0.5       -0.50   \n",
       "717        -0.500      -0.335      -0.500        -0.5        -0.5       -0.50   \n",
       "\n",
       "       1002627843  1002658094  1002848124  1002848661  ...  987424258  \\\n",
       "index                                                  ...              \n",
       "16         -0.500      -0.500      -0.500        -0.5  ...       -0.5   \n",
       "27         -0.500      -0.500      -0.500        -0.5  ...       -0.5   \n",
       "430        -0.500      -0.457      -0.492        -0.5  ...       -0.5   \n",
       "435        -0.500      -0.500      -0.464        -0.5  ...       -0.5   \n",
       "717        -0.393      -0.373      -0.500        -0.5  ...       -0.5   \n",
       "\n",
       "       987759904  987877232  987997896  988269593  988567837  988632865  \\\n",
       "index                                                                     \n",
       "16        -0.500     -0.500       -0.5     -0.500     -0.500     -0.500   \n",
       "27        -0.500     -0.500       -0.5     -0.407     -0.500     -0.361   \n",
       "430       -0.498     -0.479       -0.5     -0.500     -0.429     -0.500   \n",
       "435       -0.500     -0.500       -0.5     -0.500     -0.500     -0.324   \n",
       "717       -0.500     -0.500       -0.5     -0.489     -0.500     -0.371   \n",
       "\n",
       "       988852391  988909130  989228019  \n",
       "index                                   \n",
       "16          -0.5      -0.50     -0.500  \n",
       "27          -0.5      -0.50     -0.500  \n",
       "430         -0.5      -0.44     -0.438  \n",
       "435         -0.5      -0.50     -0.500  \n",
       "717         -0.5      -0.50     -0.500  \n",
       "\n",
       "[5 rows x 27411 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reset index\n",
    "matrix.set_index('index', inplace=True)\n",
    "matrix[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['catmaid_l1em',\n",
       " 'FlyLightSplit',\n",
       " 'FlyPNS',\n",
       " 'FlyBase',\n",
       " 'larvalbrain_axon_tract',\n",
       " 'larvalbrain_neuropil',\n",
       " 'catmaid_fafb',\n",
       " 'Konstantinides_et_al_2018_OpticLobe',\n",
       " 'neuprint_JRC_Hemibrain_1point0point1',\n",
       " 'fafb_catmaid_api',\n",
       " 'FlyCircuit',\n",
       " 'FlyLight',\n",
       " 'BrainTrap',\n",
       " 'FlyBrain_NDB',\n",
       " 'DoOR',\n",
       " 'lmb_cluster_pages_v3',\n",
       " 'lmb_cluster_pages_v2',\n",
       " 'VDRC',\n",
       " 'l1em_catmaid_api',\n",
       " 'neuronbridge',\n",
       " 'neuprint_JRC_Hemibrain_1point1',\n",
       " 'vnc1_catmaid_api']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from vfb_connect.cross_server_tools import VfbConnect\n",
    "vc = VfbConnect()\n",
    "vc.neo_query_wrapper.get_dbs()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "column_lookup = vc.neo_query_wrapper.xref_2_vfb_id(\n",
    "                db='neuprint_JRC_Hemibrain_1point1')\n",
    "row_lookup = vc.neo_query_wrapper.xref_2_vfb_id(\n",
    "                db='catmaid_fafb')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "to_drop = []\n",
    "new_columns = []\n",
    "for c in matrix.columns:\n",
    "#    if c == 'index':\n",
    "#        continue\n",
    "    if c in column_lookup.keys():\n",
    "        new_columns.append(column_lookup[c][0]['vfb_id'])\n",
    "    else:\n",
    "        to_drop.append(c)\n",
    "\n",
    "matrix.drop(columns=to_drop, inplace=True)\n",
    "\n",
    "matrix.columns = new_columns\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>VFB_jrchjv24</th>\n",
       "      <th>VFB_jrchjsbw</th>\n",
       "      <th>VFB_jrchk3az</th>\n",
       "      <th>VFB_jrchjtkl</th>\n",
       "      <th>VFB_jrchk3ck</th>\n",
       "      <th>VFB_jrchjugv</th>\n",
       "      <th>VFB_jrchk3g2</th>\n",
       "      <th>VFB_jrchk3ae</th>\n",
       "      <th>VFB_jrchjtkm</th>\n",
       "      <th>VFB_jrchk4f4</th>\n",
       "      <th>...</th>\n",
       "      <th>VFB_jrchjx1k</th>\n",
       "      <th>VFB_jrchk7cz</th>\n",
       "      <th>VFB_jrchjtyb</th>\n",
       "      <th>VFB_jrchk39j</th>\n",
       "      <th>VFB_jrchjtxm</th>\n",
       "      <th>VFB_jrchjty3</th>\n",
       "      <th>VFB_jrchk7ga</th>\n",
       "      <th>VFB_jrchk36f</th>\n",
       "      <th>VFB_jrchju27</th>\n",
       "      <th>VFB_jrchju23</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.47</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.407</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.361</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>430</th>\n",
       "      <td>-0.394</td>\n",
       "      <td>-0.282</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.457</td>\n",
       "      <td>-0.492</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.411</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.498</td>\n",
       "      <td>-0.479</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.429</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.44</td>\n",
       "      <td>-0.438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>435</th>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.464</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.324</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>717</th>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.373</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.489</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.371</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22545 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       VFB_jrchjv24  VFB_jrchjsbw  VFB_jrchk3az  VFB_jrchjtkl  VFB_jrchk3ck  \\\n",
       "index                                                                         \n",
       "16           -0.500        -0.500          -0.5          -0.5         -0.47   \n",
       "27           -0.500        -0.500          -0.5          -0.5         -0.50   \n",
       "430          -0.394        -0.282          -0.5          -0.5         -0.50   \n",
       "435          -0.500        -0.500          -0.5          -0.5         -0.50   \n",
       "717          -0.500        -0.500          -0.5          -0.5         -0.50   \n",
       "\n",
       "       VFB_jrchjugv  VFB_jrchk3g2  VFB_jrchk3ae  VFB_jrchjtkm  VFB_jrchk4f4  \\\n",
       "index                                                                         \n",
       "16           -0.500        -0.500          -0.5          -0.5        -0.500   \n",
       "27           -0.500        -0.500          -0.5          -0.5        -0.500   \n",
       "430          -0.457        -0.492          -0.5          -0.5        -0.411   \n",
       "435          -0.500        -0.464          -0.5          -0.5        -0.500   \n",
       "717          -0.373        -0.500          -0.5          -0.5        -0.500   \n",
       "\n",
       "       ...  VFB_jrchjx1k  VFB_jrchk7cz  VFB_jrchjtyb  VFB_jrchk39j  \\\n",
       "index  ...                                                           \n",
       "16     ...          -0.5        -0.500        -0.500          -0.5   \n",
       "27     ...          -0.5        -0.500        -0.500          -0.5   \n",
       "430    ...          -0.5        -0.498        -0.479          -0.5   \n",
       "435    ...          -0.5        -0.500        -0.500          -0.5   \n",
       "717    ...          -0.5        -0.500        -0.500          -0.5   \n",
       "\n",
       "       VFB_jrchjtxm  VFB_jrchjty3  VFB_jrchk7ga  VFB_jrchk36f  VFB_jrchju27  \\\n",
       "index                                                                         \n",
       "16           -0.500        -0.500        -0.500          -0.5         -0.50   \n",
       "27           -0.407        -0.500        -0.361          -0.5         -0.50   \n",
       "430          -0.500        -0.429        -0.500          -0.5         -0.44   \n",
       "435          -0.500        -0.500        -0.324          -0.5         -0.50   \n",
       "717          -0.489        -0.500        -0.371          -0.5         -0.50   \n",
       "\n",
       "       VFB_jrchju23  \n",
       "index                \n",
       "16           -0.500  \n",
       "27           -0.500  \n",
       "430          -0.438  \n",
       "435          -0.500  \n",
       "717          -0.500  \n",
       "\n",
       "[5 rows x 22545 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matrix[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type(list(row_lookup.keys())[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type(matrix['index'][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(set(matrix['index']).intersection(set(row_lookup.keys())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for r in matrix['index']:\n",
    "#    try:\n",
    "#        int(r)\n",
    "#    except:\n",
    "#        print(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11823"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rows_to_drop = [i for i in matrix.index \n",
    "                if not (i in row_lookup.keys())]\n",
    "len(rows_to_drop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type(matrix)\n",
    "matrix[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "matrix.drop(rows_to_drop, inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>VFB_jrchjv24</th>\n",
       "      <th>VFB_jrchjsbw</th>\n",
       "      <th>VFB_jrchk3az</th>\n",
       "      <th>VFB_jrchjtkl</th>\n",
       "      <th>VFB_jrchk3ck</th>\n",
       "      <th>VFB_jrchjugv</th>\n",
       "      <th>VFB_jrchk3g2</th>\n",
       "      <th>VFB_jrchk3ae</th>\n",
       "      <th>VFB_jrchjtkm</th>\n",
       "      <th>VFB_jrchk4f4</th>\n",
       "      <th>...</th>\n",
       "      <th>VFB_jrchjx1k</th>\n",
       "      <th>VFB_jrchk7cz</th>\n",
       "      <th>VFB_jrchjtyb</th>\n",
       "      <th>VFB_jrchk39j</th>\n",
       "      <th>VFB_jrchjtxm</th>\n",
       "      <th>VFB_jrchjty3</th>\n",
       "      <th>VFB_jrchk7ga</th>\n",
       "      <th>VFB_jrchk36f</th>\n",
       "      <th>VFB_jrchju27</th>\n",
       "      <th>VFB_jrchju23</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6510</th>\n",
       "      <td>0.066</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.257</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6582</th>\n",
       "      <td>0.110</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.495</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.219</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6618</th>\n",
       "      <td>0.222</td>\n",
       "      <td>-0.486</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.466</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.223</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6630</th>\n",
       "      <td>0.124</td>\n",
       "      <td>-0.495</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.475</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6662</th>\n",
       "      <td>0.061</td>\n",
       "      <td>-0.496</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11524386</th>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11524475</th>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11543538</th>\n",
       "      <td>-0.418</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.47</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.461</td>\n",
       "      <td>-0.411</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11545702</th>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11547665</th>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1834 rows × 22545 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          VFB_jrchjv24  VFB_jrchjsbw  VFB_jrchk3az  VFB_jrchjtkl  \\\n",
       "index                                                              \n",
       "6510             0.066        -0.500         -0.50          -0.5   \n",
       "6582             0.110        -0.500         -0.50          -0.5   \n",
       "6618             0.222        -0.486         -0.50          -0.5   \n",
       "6630             0.124        -0.495         -0.50          -0.5   \n",
       "6662             0.061        -0.496         -0.50          -0.5   \n",
       "...                ...           ...           ...           ...   \n",
       "11524386        -0.500        -0.500         -0.50          -0.5   \n",
       "11524475        -0.500        -0.500         -0.50          -0.5   \n",
       "11543538        -0.418        -0.500         -0.47          -0.5   \n",
       "11545702        -0.500        -0.500         -0.50          -0.5   \n",
       "11547665        -0.500        -0.500         -0.50          -0.5   \n",
       "\n",
       "          VFB_jrchk3ck  VFB_jrchjugv  VFB_jrchk3g2  VFB_jrchk3ae  \\\n",
       "index                                                              \n",
       "6510              -0.5          -0.5        -0.500        -0.500   \n",
       "6582              -0.5          -0.5        -0.500        -0.495   \n",
       "6618              -0.5          -0.5        -0.500        -0.466   \n",
       "6630              -0.5          -0.5        -0.500        -0.500   \n",
       "6662              -0.5          -0.5        -0.500        -0.500   \n",
       "...                ...           ...           ...           ...   \n",
       "11524386          -0.5          -0.5        -0.500        -0.500   \n",
       "11524475          -0.5          -0.5        -0.500        -0.500   \n",
       "11543538          -0.5          -0.5        -0.461        -0.411   \n",
       "11545702          -0.5          -0.5        -0.500        -0.500   \n",
       "11547665          -0.5          -0.5        -0.500        -0.500   \n",
       "\n",
       "          VFB_jrchjtkm  VFB_jrchk4f4  ...  VFB_jrchjx1k  VFB_jrchk7cz  \\\n",
       "index                                 ...                               \n",
       "6510              -0.5          -0.5  ...          -0.5          -0.5   \n",
       "6582              -0.5          -0.5  ...          -0.5          -0.5   \n",
       "6618              -0.5          -0.5  ...          -0.5          -0.5   \n",
       "6630              -0.5          -0.5  ...          -0.5          -0.5   \n",
       "6662              -0.5          -0.5  ...          -0.5          -0.5   \n",
       "...                ...           ...  ...           ...           ...   \n",
       "11524386          -0.5          -0.5  ...          -0.5          -0.5   \n",
       "11524475          -0.5          -0.5  ...          -0.5          -0.5   \n",
       "11543538          -0.5          -0.5  ...          -0.5          -0.5   \n",
       "11545702          -0.5          -0.5  ...          -0.5          -0.5   \n",
       "11547665          -0.5          -0.5  ...          -0.5          -0.5   \n",
       "\n",
       "          VFB_jrchjtyb  VFB_jrchk39j  VFB_jrchjtxm  VFB_jrchjty3  \\\n",
       "index                                                              \n",
       "6510              -0.5          -0.5          -0.5          -0.5   \n",
       "6582              -0.5          -0.5          -0.5          -0.5   \n",
       "6618              -0.5          -0.5          -0.5          -0.5   \n",
       "6630              -0.5          -0.5          -0.5          -0.5   \n",
       "6662              -0.5          -0.5          -0.5          -0.5   \n",
       "...                ...           ...           ...           ...   \n",
       "11524386          -0.5          -0.5          -0.5          -0.5   \n",
       "11524475          -0.5          -0.5          -0.5          -0.5   \n",
       "11543538          -0.5          -0.5          -0.5          -0.5   \n",
       "11545702          -0.5          -0.5          -0.5          -0.5   \n",
       "11547665          -0.5          -0.5          -0.5          -0.5   \n",
       "\n",
       "          VFB_jrchk7ga  VFB_jrchk36f  VFB_jrchju27  VFB_jrchju23  \n",
       "index                                                             \n",
       "6510              -0.5        -0.257          -0.5          -0.5  \n",
       "6582              -0.5        -0.219          -0.5          -0.5  \n",
       "6618              -0.5        -0.223          -0.5          -0.5  \n",
       "6630              -0.5        -0.475          -0.5          -0.5  \n",
       "6662              -0.5        -0.500          -0.5          -0.5  \n",
       "...                ...           ...           ...           ...  \n",
       "11524386          -0.5        -0.500          -0.5          -0.5  \n",
       "11524475          -0.5        -0.500          -0.5          -0.5  \n",
       "11543538          -0.5        -0.500          -0.5          -0.5  \n",
       "11545702          -0.5        -0.500          -0.5          -0.5  \n",
       "11547665          -0.5        -0.500          -0.5          -0.5  \n",
       "\n",
       "[1834 rows x 22545 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "par"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>VFB_jrchjv24</th>\n",
       "      <th>VFB_jrchjsbw</th>\n",
       "      <th>VFB_jrchk3az</th>\n",
       "      <th>VFB_jrchjtkl</th>\n",
       "      <th>VFB_jrchk3ck</th>\n",
       "      <th>VFB_jrchjugv</th>\n",
       "      <th>VFB_jrchk3g2</th>\n",
       "      <th>VFB_jrchk3ae</th>\n",
       "      <th>VFB_jrchjtkm</th>\n",
       "      <th>VFB_jrchk4f4</th>\n",
       "      <th>...</th>\n",
       "      <th>VFB_jrchjx1k</th>\n",
       "      <th>VFB_jrchk7cz</th>\n",
       "      <th>VFB_jrchjtyb</th>\n",
       "      <th>VFB_jrchk39j</th>\n",
       "      <th>VFB_jrchjtxm</th>\n",
       "      <th>VFB_jrchjty3</th>\n",
       "      <th>VFB_jrchk7ga</th>\n",
       "      <th>VFB_jrchk36f</th>\n",
       "      <th>VFB_jrchju27</th>\n",
       "      <th>VFB_jrchju23</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>VFB_0010003u</th>\n",
       "      <td>0.066</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.257</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VFB_0010003v</th>\n",
       "      <td>0.110</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.495</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.219</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VFB_0010003w</th>\n",
       "      <td>0.222</td>\n",
       "      <td>-0.486</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.466</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.223</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VFB_0010003x</th>\n",
       "      <td>0.124</td>\n",
       "      <td>-0.495</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.475</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VFB_0010003y</th>\n",
       "      <td>0.061</td>\n",
       "      <td>-0.496</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VFB_00101376</th>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VFB_00101377</th>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VFB_001001dp</th>\n",
       "      <td>-0.418</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.47</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.461</td>\n",
       "      <td>-0.411</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VFB_00101378</th>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VFB_00101283</th>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1834 rows × 22545 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              VFB_jrchjv24  VFB_jrchjsbw  VFB_jrchk3az  VFB_jrchjtkl  \\\n",
       "index                                                                  \n",
       "VFB_0010003u         0.066        -0.500         -0.50          -0.5   \n",
       "VFB_0010003v         0.110        -0.500         -0.50          -0.5   \n",
       "VFB_0010003w         0.222        -0.486         -0.50          -0.5   \n",
       "VFB_0010003x         0.124        -0.495         -0.50          -0.5   \n",
       "VFB_0010003y         0.061        -0.496         -0.50          -0.5   \n",
       "...                    ...           ...           ...           ...   \n",
       "VFB_00101376        -0.500        -0.500         -0.50          -0.5   \n",
       "VFB_00101377        -0.500        -0.500         -0.50          -0.5   \n",
       "VFB_001001dp        -0.418        -0.500         -0.47          -0.5   \n",
       "VFB_00101378        -0.500        -0.500         -0.50          -0.5   \n",
       "VFB_00101283        -0.500        -0.500         -0.50          -0.5   \n",
       "\n",
       "              VFB_jrchk3ck  VFB_jrchjugv  VFB_jrchk3g2  VFB_jrchk3ae  \\\n",
       "index                                                                  \n",
       "VFB_0010003u          -0.5          -0.5        -0.500        -0.500   \n",
       "VFB_0010003v          -0.5          -0.5        -0.500        -0.495   \n",
       "VFB_0010003w          -0.5          -0.5        -0.500        -0.466   \n",
       "VFB_0010003x          -0.5          -0.5        -0.500        -0.500   \n",
       "VFB_0010003y          -0.5          -0.5        -0.500        -0.500   \n",
       "...                    ...           ...           ...           ...   \n",
       "VFB_00101376          -0.5          -0.5        -0.500        -0.500   \n",
       "VFB_00101377          -0.5          -0.5        -0.500        -0.500   \n",
       "VFB_001001dp          -0.5          -0.5        -0.461        -0.411   \n",
       "VFB_00101378          -0.5          -0.5        -0.500        -0.500   \n",
       "VFB_00101283          -0.5          -0.5        -0.500        -0.500   \n",
       "\n",
       "              VFB_jrchjtkm  VFB_jrchk4f4  ...  VFB_jrchjx1k  VFB_jrchk7cz  \\\n",
       "index                                     ...                               \n",
       "VFB_0010003u          -0.5          -0.5  ...          -0.5          -0.5   \n",
       "VFB_0010003v          -0.5          -0.5  ...          -0.5          -0.5   \n",
       "VFB_0010003w          -0.5          -0.5  ...          -0.5          -0.5   \n",
       "VFB_0010003x          -0.5          -0.5  ...          -0.5          -0.5   \n",
       "VFB_0010003y          -0.5          -0.5  ...          -0.5          -0.5   \n",
       "...                    ...           ...  ...           ...           ...   \n",
       "VFB_00101376          -0.5          -0.5  ...          -0.5          -0.5   \n",
       "VFB_00101377          -0.5          -0.5  ...          -0.5          -0.5   \n",
       "VFB_001001dp          -0.5          -0.5  ...          -0.5          -0.5   \n",
       "VFB_00101378          -0.5          -0.5  ...          -0.5          -0.5   \n",
       "VFB_00101283          -0.5          -0.5  ...          -0.5          -0.5   \n",
       "\n",
       "              VFB_jrchjtyb  VFB_jrchk39j  VFB_jrchjtxm  VFB_jrchjty3  \\\n",
       "index                                                                  \n",
       "VFB_0010003u          -0.5          -0.5          -0.5          -0.5   \n",
       "VFB_0010003v          -0.5          -0.5          -0.5          -0.5   \n",
       "VFB_0010003w          -0.5          -0.5          -0.5          -0.5   \n",
       "VFB_0010003x          -0.5          -0.5          -0.5          -0.5   \n",
       "VFB_0010003y          -0.5          -0.5          -0.5          -0.5   \n",
       "...                    ...           ...           ...           ...   \n",
       "VFB_00101376          -0.5          -0.5          -0.5          -0.5   \n",
       "VFB_00101377          -0.5          -0.5          -0.5          -0.5   \n",
       "VFB_001001dp          -0.5          -0.5          -0.5          -0.5   \n",
       "VFB_00101378          -0.5          -0.5          -0.5          -0.5   \n",
       "VFB_00101283          -0.5          -0.5          -0.5          -0.5   \n",
       "\n",
       "              VFB_jrchk7ga  VFB_jrchk36f  VFB_jrchju27  VFB_jrchju23  \n",
       "index                                                                 \n",
       "VFB_0010003u          -0.5        -0.257          -0.5          -0.5  \n",
       "VFB_0010003v          -0.5        -0.219          -0.5          -0.5  \n",
       "VFB_0010003w          -0.5        -0.223          -0.5          -0.5  \n",
       "VFB_0010003x          -0.5        -0.475          -0.5          -0.5  \n",
       "VFB_0010003y          -0.5        -0.500          -0.5          -0.5  \n",
       "...                    ...           ...           ...           ...  \n",
       "VFB_00101376          -0.5        -0.500          -0.5          -0.5  \n",
       "VFB_00101377          -0.5        -0.500          -0.5          -0.5  \n",
       "VFB_001001dp          -0.5        -0.500          -0.5          -0.5  \n",
       "VFB_00101378          -0.5        -0.500          -0.5          -0.5  \n",
       "VFB_00101283          -0.5        -0.500          -0.5          -0.5  \n",
       "\n",
       "[1834 rows x 22545 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>1001453586</th>\n",
       "      <th>1001971876</th>\n",
       "      <th>1002360103</th>\n",
       "      <th>1002507131</th>\n",
       "      <th>1002507159</th>\n",
       "      <th>1002507170</th>\n",
       "      <th>1002627843</th>\n",
       "      <th>1002658094</th>\n",
       "      <th>1002848124</th>\n",
       "      <th>...</th>\n",
       "      <th>987424258</th>\n",
       "      <th>987759904</th>\n",
       "      <th>987877232</th>\n",
       "      <th>987997896</th>\n",
       "      <th>988269593</th>\n",
       "      <th>988567837</th>\n",
       "      <th>988632865</th>\n",
       "      <th>988852391</th>\n",
       "      <th>988909130</th>\n",
       "      <th>989228019</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>16</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.47</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>27</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.485</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.407</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.361</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>430</td>\n",
       "      <td>-0.394</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.282</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.457</td>\n",
       "      <td>-0.492</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.498</td>\n",
       "      <td>-0.479</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.429</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.44</td>\n",
       "      <td>-0.438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>435</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.464</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.324</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>717</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.335</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.393</td>\n",
       "      <td>-0.373</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.489</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.371</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13652</th>\n",
       "      <td>16521809</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13653</th>\n",
       "      <td>16521909</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.150</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13654</th>\n",
       "      <td>16527823</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.058</td>\n",
       "      <td>-0.398</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13655</th>\n",
       "      <td>16529568</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.464</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13656</th>\n",
       "      <td>16529586</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.117</td>\n",
       "      <td>-0.397</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>-0.500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13657 rows × 27412 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          index  1001453586  1001971876  1002360103  1002507131  1002507159  \\\n",
       "0            16      -0.500      -0.500      -0.500        -0.5        -0.5   \n",
       "1            27      -0.500      -0.485      -0.500        -0.5        -0.5   \n",
       "2           430      -0.394      -0.500      -0.282        -0.5        -0.5   \n",
       "3           435      -0.500      -0.500      -0.500        -0.5        -0.5   \n",
       "4           717      -0.500      -0.335      -0.500        -0.5        -0.5   \n",
       "...         ...         ...         ...         ...         ...         ...   \n",
       "13652  16521809      -0.500      -0.500      -0.500        -0.5        -0.5   \n",
       "13653  16521909      -0.500      -0.500      -0.500        -0.5        -0.5   \n",
       "13654  16527823      -0.500      -0.500      -0.500        -0.5        -0.5   \n",
       "13655  16529568      -0.500      -0.500      -0.500        -0.5        -0.5   \n",
       "13656  16529586      -0.500      -0.500      -0.500        -0.5        -0.5   \n",
       "\n",
       "       1002507170  1002627843  1002658094  1002848124  ...  987424258  \\\n",
       "0           -0.47      -0.500      -0.500      -0.500  ...     -0.500   \n",
       "1           -0.50      -0.500      -0.500      -0.500  ...     -0.500   \n",
       "2           -0.50      -0.500      -0.457      -0.492  ...     -0.500   \n",
       "3           -0.50      -0.500      -0.500      -0.464  ...     -0.500   \n",
       "4           -0.50      -0.393      -0.373      -0.500  ...     -0.500   \n",
       "...           ...         ...         ...         ...  ...        ...   \n",
       "13652       -0.50      -0.500      -0.500      -0.500  ...     -0.500   \n",
       "13653       -0.50      -0.500      -0.500      -0.500  ...     -0.150   \n",
       "13654       -0.50      -0.500      -0.500      -0.500  ...     -0.058   \n",
       "13655       -0.50      -0.500      -0.500      -0.500  ...     -0.464   \n",
       "13656       -0.50      -0.500      -0.500      -0.500  ...     -0.117   \n",
       "\n",
       "       987759904  987877232  987997896  988269593  988567837  988632865  \\\n",
       "0         -0.500     -0.500       -0.5     -0.500     -0.500     -0.500   \n",
       "1         -0.500     -0.500       -0.5     -0.407     -0.500     -0.361   \n",
       "2         -0.498     -0.479       -0.5     -0.500     -0.429     -0.500   \n",
       "3         -0.500     -0.500       -0.5     -0.500     -0.500     -0.324   \n",
       "4         -0.500     -0.500       -0.5     -0.489     -0.500     -0.371   \n",
       "...          ...        ...        ...        ...        ...        ...   \n",
       "13652     -0.500     -0.500       -0.5     -0.500     -0.500     -0.500   \n",
       "13653     -0.500     -0.500       -0.5     -0.500     -0.500     -0.500   \n",
       "13654     -0.398     -0.500       -0.5     -0.500     -0.500     -0.500   \n",
       "13655     -0.500     -0.500       -0.5     -0.500     -0.500     -0.500   \n",
       "13656     -0.397     -0.500       -0.5     -0.500     -0.500     -0.500   \n",
       "\n",
       "       988852391  988909130  989228019  \n",
       "0           -0.5      -0.50     -0.500  \n",
       "1           -0.5      -0.50     -0.500  \n",
       "2           -0.5      -0.44     -0.438  \n",
       "3           -0.5      -0.50     -0.500  \n",
       "4           -0.5      -0.50     -0.500  \n",
       "...          ...        ...        ...  \n",
       "13652       -0.5      -0.50     -0.500  \n",
       "13653       -0.5      -0.50     -0.500  \n",
       "13654       -0.5      -0.50     -0.500  \n",
       "13655       -0.5      -0.50     -0.500  \n",
       "13656       -0.5      -0.50     -0.500  \n",
       "\n",
       "[13657 rows x 27412 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matrix.reindex()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "isinstance(matrix['VFB_jrchjv24'][1], float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "test_matrix = pd.DataFrame.from_records([{'a': 1.0, 'b': 2.1, 'c': 3.2},\n",
    "                                         {'a': 3.0, 'b': 1.1, 'c': -1.0}])\n",
    "test_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy\n",
    "\n",
    "def threshold(x):\n",
    "    if type(x) is float and (x > 2):\n",
    "        return numpy.NaN\n",
    "    else:\n",
    "        return x\n",
    "\n",
    "reduced_test_matrix = test_matrix.applymap(threshold)\n",
    "print(reduced_test_matrix)\n",
    "sparse_matrix = reduced_test_matrix.astype(pd.SparseDtype(\"float\", numpy.NaN))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduced_test_matrix.drop([0], inplace=True)\n",
    "reduced_test_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy\n",
    "import pandas as pd\n",
    "def threshold(x):\n",
    "    if (type(x) is float) and (x < 0.3):\n",
    "        return numpy.NaN\n",
    "    else:\n",
    "        return x\n",
    "reduced_matrix = matrix.applymap(threshold)\n",
    "sparse_matrix = reduced_matrix.astype(pd.SparseDtype(\"float\", numpy.NaN))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduced_matrix.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(reduced_matrix[0:5])\n",
    "print(sparse_matrix[0:5])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduced_matrix.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[(i,x) for i,x, in reduced_matrix['VFB_jrchjv24'].items()\n",
    " if x]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>1001453586</th>\n",
       "      <th>1001971876</th>\n",
       "      <th>1002360103</th>\n",
       "      <th>1002507131</th>\n",
       "      <th>1002507159</th>\n",
       "      <th>1002507170</th>\n",
       "      <th>1002627843</th>\n",
       "      <th>1002658094</th>\n",
       "      <th>1002848124</th>\n",
       "      <th>...</th>\n",
       "      <th>987424258</th>\n",
       "      <th>987759904</th>\n",
       "      <th>987877232</th>\n",
       "      <th>987997896</th>\n",
       "      <th>988269593</th>\n",
       "      <th>988567837</th>\n",
       "      <th>988632865</th>\n",
       "      <th>988852391</th>\n",
       "      <th>988909130</th>\n",
       "      <th>989228019</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>27</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>430</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>435</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>717</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13652</th>\n",
       "      <td>16521809</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13653</th>\n",
       "      <td>16521909</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13654</th>\n",
       "      <td>16527823</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13655</th>\n",
       "      <td>16529568</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13656</th>\n",
       "      <td>16529586</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13657 rows × 27412 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          index  1001453586  1001971876  1002360103  1002507131  1002507159  \\\n",
       "0            16         NaN         NaN         NaN         NaN         NaN   \n",
       "1            27         NaN         NaN         NaN         NaN         NaN   \n",
       "2           430         NaN         NaN         NaN         NaN         NaN   \n",
       "3           435         NaN         NaN         NaN         NaN         NaN   \n",
       "4           717         NaN         NaN         NaN         NaN         NaN   \n",
       "...         ...         ...         ...         ...         ...         ...   \n",
       "13652  16521809         NaN         NaN         NaN         NaN         NaN   \n",
       "13653  16521909         NaN         NaN         NaN         NaN         NaN   \n",
       "13654  16527823         NaN         NaN         NaN         NaN         NaN   \n",
       "13655  16529568         NaN         NaN         NaN         NaN         NaN   \n",
       "13656  16529586         NaN         NaN         NaN         NaN         NaN   \n",
       "\n",
       "       1002507170  1002627843  1002658094  1002848124  ...  987424258  \\\n",
       "0             NaN         NaN         NaN         NaN  ...        NaN   \n",
       "1             NaN         NaN         NaN         NaN  ...        NaN   \n",
       "2             NaN         NaN         NaN         NaN  ...        NaN   \n",
       "3             NaN         NaN         NaN         NaN  ...        NaN   \n",
       "4             NaN         NaN         NaN         NaN  ...        NaN   \n",
       "...           ...         ...         ...         ...  ...        ...   \n",
       "13652         NaN         NaN         NaN         NaN  ...        NaN   \n",
       "13653         NaN         NaN         NaN         NaN  ...        NaN   \n",
       "13654         NaN         NaN         NaN         NaN  ...        NaN   \n",
       "13655         NaN         NaN         NaN         NaN  ...        NaN   \n",
       "13656         NaN         NaN         NaN         NaN  ...        NaN   \n",
       "\n",
       "       987759904  987877232  987997896  988269593  988567837  988632865  \\\n",
       "0            NaN        NaN        NaN        NaN        NaN        NaN   \n",
       "1            NaN        NaN        NaN        NaN        NaN        NaN   \n",
       "2            NaN        NaN        NaN        NaN        NaN        NaN   \n",
       "3            NaN        NaN        NaN        NaN        NaN        NaN   \n",
       "4            NaN        NaN        NaN        NaN        NaN        NaN   \n",
       "...          ...        ...        ...        ...        ...        ...   \n",
       "13652        NaN        NaN        NaN        NaN        NaN        NaN   \n",
       "13653        NaN        NaN        NaN        NaN        NaN        NaN   \n",
       "13654        NaN        NaN        NaN        NaN        NaN        NaN   \n",
       "13655        NaN        NaN        NaN        NaN        NaN        NaN   \n",
       "13656        NaN        NaN        NaN        NaN        NaN        NaN   \n",
       "\n",
       "       988852391  988909130  989228019  \n",
       "0            NaN        NaN        NaN  \n",
       "1            NaN        NaN        NaN  \n",
       "2            NaN        NaN        NaN  \n",
       "3            NaN        NaN        NaN  \n",
       "4            NaN        NaN        NaN  \n",
       "...          ...        ...        ...  \n",
       "13652        NaN        NaN        NaN  \n",
       "13653        NaN        NaN        NaN  \n",
       "13654        NaN        NaN        NaN  \n",
       "13655        NaN        NaN        NaN  \n",
       "13656        NaN        NaN        NaN  \n",
       "\n",
       "[13657 rows x 27412 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reduced_matrix.reindex()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>VFB_jrchjv24</th>\n",
       "      <th>VFB_jrchjsbw</th>\n",
       "      <th>VFB_jrchk3az</th>\n",
       "      <th>VFB_jrchjtkl</th>\n",
       "      <th>VFB_jrchk3ck</th>\n",
       "      <th>VFB_jrchjugv</th>\n",
       "      <th>VFB_jrchk3g2</th>\n",
       "      <th>VFB_jrchk3ae</th>\n",
       "      <th>VFB_jrchjtkm</th>\n",
       "      <th>VFB_jrchk4f4</th>\n",
       "      <th>...</th>\n",
       "      <th>VFB_jrchjx1k</th>\n",
       "      <th>VFB_jrchk7cz</th>\n",
       "      <th>VFB_jrchjtyb</th>\n",
       "      <th>VFB_jrchk39j</th>\n",
       "      <th>VFB_jrchjtxm</th>\n",
       "      <th>VFB_jrchjty3</th>\n",
       "      <th>VFB_jrchk7ga</th>\n",
       "      <th>VFB_jrchk36f</th>\n",
       "      <th>VFB_jrchju27</th>\n",
       "      <th>VFB_jrchju23</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1829</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1830</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1831</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1832</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1833</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1834 rows × 22545 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      VFB_jrchjv24  VFB_jrchjsbw  VFB_jrchk3az  VFB_jrchjtkl  VFB_jrchk3ck  \\\n",
       "0              NaN           NaN           NaN           NaN           NaN   \n",
       "1              NaN           NaN           NaN           NaN           NaN   \n",
       "2              NaN           NaN           NaN           NaN           NaN   \n",
       "3              NaN           NaN           NaN           NaN           NaN   \n",
       "4              NaN           NaN           NaN           NaN           NaN   \n",
       "...            ...           ...           ...           ...           ...   \n",
       "1829           NaN           NaN           NaN           NaN           NaN   \n",
       "1830           NaN           NaN           NaN           NaN           NaN   \n",
       "1831           NaN           NaN           NaN           NaN           NaN   \n",
       "1832           NaN           NaN           NaN           NaN           NaN   \n",
       "1833           NaN           NaN           NaN           NaN           NaN   \n",
       "\n",
       "      VFB_jrchjugv  VFB_jrchk3g2  VFB_jrchk3ae  VFB_jrchjtkm  VFB_jrchk4f4  \\\n",
       "0              NaN           NaN           NaN           NaN           NaN   \n",
       "1              NaN           NaN           NaN           NaN           NaN   \n",
       "2              NaN           NaN           NaN           NaN           NaN   \n",
       "3              NaN           NaN           NaN           NaN           NaN   \n",
       "4              NaN           NaN           NaN           NaN           NaN   \n",
       "...            ...           ...           ...           ...           ...   \n",
       "1829           NaN           NaN           NaN           NaN           NaN   \n",
       "1830           NaN           NaN           NaN           NaN           NaN   \n",
       "1831           NaN           NaN           NaN           NaN           NaN   \n",
       "1832           NaN           NaN           NaN           NaN           NaN   \n",
       "1833           NaN           NaN           NaN           NaN           NaN   \n",
       "\n",
       "      ...  VFB_jrchjx1k  VFB_jrchk7cz  VFB_jrchjtyb  VFB_jrchk39j  \\\n",
       "0     ...           NaN           NaN           NaN           NaN   \n",
       "1     ...           NaN           NaN           NaN           NaN   \n",
       "2     ...           NaN           NaN           NaN           NaN   \n",
       "3     ...           NaN           NaN           NaN           NaN   \n",
       "4     ...           NaN           NaN           NaN           NaN   \n",
       "...   ...           ...           ...           ...           ...   \n",
       "1829  ...           NaN           NaN           NaN           NaN   \n",
       "1830  ...           NaN           NaN           NaN           NaN   \n",
       "1831  ...           NaN           NaN           NaN           NaN   \n",
       "1832  ...           NaN           NaN           NaN           NaN   \n",
       "1833  ...           NaN           NaN           NaN           NaN   \n",
       "\n",
       "      VFB_jrchjtxm  VFB_jrchjty3  VFB_jrchk7ga  VFB_jrchk36f  VFB_jrchju27  \\\n",
       "0              NaN           NaN           NaN           NaN           NaN   \n",
       "1              NaN           NaN           NaN           NaN           NaN   \n",
       "2              NaN           NaN           NaN           NaN           NaN   \n",
       "3              NaN           NaN           NaN           NaN           NaN   \n",
       "4              NaN           NaN           NaN           NaN           NaN   \n",
       "...            ...           ...           ...           ...           ...   \n",
       "1829           NaN           NaN           NaN           NaN           NaN   \n",
       "1830           NaN           NaN           NaN           NaN           NaN   \n",
       "1831           NaN           NaN           NaN           NaN           NaN   \n",
       "1832           NaN           NaN           NaN           NaN           NaN   \n",
       "1833           NaN           NaN           NaN           NaN           NaN   \n",
       "\n",
       "      VFB_jrchju23  \n",
       "0              NaN  \n",
       "1              NaN  \n",
       "2              NaN  \n",
       "3              NaN  \n",
       "4              NaN  \n",
       "...            ...  \n",
       "1829           NaN  \n",
       "1830           NaN  \n",
       "1831           NaN  \n",
       "1832           NaN  \n",
       "1833           NaN  \n",
       "\n",
       "[1834 rows x 22545 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "minmat = feather.read_dataframe('../src/reduced_matrix.feather')\n",
    "minmat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "desc = minmat.describe()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>VFB_jrchjv24</th>\n",
       "      <th>VFB_jrchjsbw</th>\n",
       "      <th>VFB_jrchk3az</th>\n",
       "      <th>VFB_jrchjtkl</th>\n",
       "      <th>VFB_jrchk3ck</th>\n",
       "      <th>VFB_jrchjugv</th>\n",
       "      <th>VFB_jrchk3g2</th>\n",
       "      <th>VFB_jrchk3ae</th>\n",
       "      <th>VFB_jrchjtkm</th>\n",
       "      <th>VFB_jrchk4f4</th>\n",
       "      <th>...</th>\n",
       "      <th>VFB_jrchjx1k</th>\n",
       "      <th>VFB_jrchk7cz</th>\n",
       "      <th>VFB_jrchjtyb</th>\n",
       "      <th>VFB_jrchk39j</th>\n",
       "      <th>VFB_jrchjtxm</th>\n",
       "      <th>VFB_jrchjty3</th>\n",
       "      <th>VFB_jrchk7ga</th>\n",
       "      <th>VFB_jrchk36f</th>\n",
       "      <th>VFB_jrchju27</th>\n",
       "      <th>VFB_jrchju23</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.336250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.051403</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.305000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.308750</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.313500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.341000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.413000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 22545 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       VFB_jrchjv24  VFB_jrchjsbw  VFB_jrchk3az  VFB_jrchjtkl  VFB_jrchk3ck  \\\n",
       "count      4.000000           0.0           0.0           0.0           0.0   \n",
       "mean       0.336250           NaN           NaN           NaN           NaN   \n",
       "std        0.051403           NaN           NaN           NaN           NaN   \n",
       "min        0.305000           NaN           NaN           NaN           NaN   \n",
       "25%        0.308750           NaN           NaN           NaN           NaN   \n",
       "50%        0.313500           NaN           NaN           NaN           NaN   \n",
       "75%        0.341000           NaN           NaN           NaN           NaN   \n",
       "max        0.413000           NaN           NaN           NaN           NaN   \n",
       "\n",
       "       VFB_jrchjugv  VFB_jrchk3g2  VFB_jrchk3ae  VFB_jrchjtkm  VFB_jrchk4f4  \\\n",
       "count           0.0           0.0           0.0           0.0           0.0   \n",
       "mean            NaN           NaN           NaN           NaN           NaN   \n",
       "std             NaN           NaN           NaN           NaN           NaN   \n",
       "min             NaN           NaN           NaN           NaN           NaN   \n",
       "25%             NaN           NaN           NaN           NaN           NaN   \n",
       "50%             NaN           NaN           NaN           NaN           NaN   \n",
       "75%             NaN           NaN           NaN           NaN           NaN   \n",
       "max             NaN           NaN           NaN           NaN           NaN   \n",
       "\n",
       "       ...  VFB_jrchjx1k  VFB_jrchk7cz  VFB_jrchjtyb  VFB_jrchk39j  \\\n",
       "count  ...           0.0           0.0           0.0           0.0   \n",
       "mean   ...           NaN           NaN           NaN           NaN   \n",
       "std    ...           NaN           NaN           NaN           NaN   \n",
       "min    ...           NaN           NaN           NaN           NaN   \n",
       "25%    ...           NaN           NaN           NaN           NaN   \n",
       "50%    ...           NaN           NaN           NaN           NaN   \n",
       "75%    ...           NaN           NaN           NaN           NaN   \n",
       "max    ...           NaN           NaN           NaN           NaN   \n",
       "\n",
       "       VFB_jrchjtxm  VFB_jrchjty3  VFB_jrchk7ga  VFB_jrchk36f  VFB_jrchju27  \\\n",
       "count           0.0           0.0           0.0           0.0           0.0   \n",
       "mean            NaN           NaN           NaN           NaN           NaN   \n",
       "std             NaN           NaN           NaN           NaN           NaN   \n",
       "min             NaN           NaN           NaN           NaN           NaN   \n",
       "25%             NaN           NaN           NaN           NaN           NaN   \n",
       "50%             NaN           NaN           NaN           NaN           NaN   \n",
       "75%             NaN           NaN           NaN           NaN           NaN   \n",
       "max             NaN           NaN           NaN           NaN           NaN   \n",
       "\n",
       "       VFB_jrchju23  \n",
       "count           0.0  \n",
       "mean            NaN  \n",
       "std             NaN  \n",
       "min             NaN  \n",
       "25%             NaN  \n",
       "50%             NaN  \n",
       "75%             NaN  \n",
       "max             NaN  \n",
       "\n",
       "[8 rows x 22545 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "desc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    22545.000000\n",
       "mean         3.906409\n",
       "std         21.884506\n",
       "min          0.000000\n",
       "25%          0.000000\n",
       "50%          0.000000\n",
       "75%          0.000000\n",
       "max        317.000000\n",
       "Name: count, dtype: float64"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "desc.loc['count'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}