Test_domains.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "470668bf",
   "metadata": {},
   "source": [
    "# Test model performance on adhesin motifs not previously encountered"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e863cda",
   "metadata": {},
   "source": [
    "### Count sequences per category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3ef7131b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from Bio import SeqIO\n",
    "import os\n",
    "import copy\n",
    "\n",
    "ADH_60 = './data/60_similarity/positive/adhesins.fasta'\n",
    "ADH_25 = './data/25_similarity/positive/adhesins.fasta'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "21cbf656",
   "metadata": {},
   "outputs": [],
   "source": [
    "# initiate dictionaries with all sequences from all motifs\n",
    "sequence_dictionary_60 = {}\n",
    "dir_ = './data/raw_sequences/'\n",
    "for file in os.listdir(dir_):\n",
    "    if len(file) < 11:\n",
    "        motif_name = file.split('.')[0]\n",
    "        motif_sequences = list(SeqIO.parse(os.path.join(dir_, file), 'fasta'))\n",
    "        sequence_dictionary_60[motif_name] = {seq.id:0 for seq in motif_sequences}\n",
    "sequence_dictionary_25 = copy.deepcopy(sequence_dictionary_60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a223faa7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# fill dictionaries\n",
    "adh_25_sequences = list(SeqIO.parse(ADH_25, 'fasta'))\n",
    "for seq in adh_25_sequences:\n",
    "    for motif in sequence_dictionary_25:\n",
    "        if seq.id in sequence_dictionary_25[motif]:\n",
    "            sequence_dictionary_25[motif][seq.id] = 1\n",
    "            \n",
    "adh_60_sequences = list(SeqIO.parse(ADH_60, 'fasta'))\n",
    "for seq in adh_60_sequences:\n",
    "    for motif in sequence_dictionary_60:\n",
    "        if seq.id in sequence_dictionary_60[motif]:\n",
    "            sequence_dictionary_60[motif][seq.id] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "98403c6e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Protein count per motif at 60% and 25% cleaning:\n",
      "PF08363 44 15\n",
      "PF05737 484 153\n",
      "PF08829 2 2\n",
      "PF18220 7 5\n",
      "PF17480 1 1\n",
      "PF12799 5932 1719\n",
      "PF18573 313 158\n",
      "PF03212 431 136\n",
      "PF07979 1 1\n",
      "PF02216 5 4\n",
      "PF15403 1 1\n",
      "PF18651 121 37\n",
      "PF05658 1299 414\n",
      "PF08341 409 129\n",
      "PF07675 288 76\n",
      "PF18304 5 1\n",
      "PF09160 20 5\n",
      "PF18652 73 22\n",
      "PF09403 16 10\n",
      "PF18873 6 2\n",
      "PF18483 491 234\n",
      "PF10425 48 33\n",
      "PF11966 7 1\n",
      "PF15401 7 5\n",
      "PF07691 2089 569\n"
     ]
    }
   ],
   "source": [
    "# count\n",
    "print('Protein count per motif at 60% and 25% cleaning:')\n",
    "for motif in sequence_dictionary_25:\n",
    "    count_60 = sum(sequence_dictionary_60[motif].values())\n",
    "    count_25 = sum(sequence_dictionary_25[motif].values())\n",
    "    print(f'{motif} {count_60} {count_25}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81d26164",
   "metadata": {},
   "source": [
    "### Asses model perfomance on different datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "10cb251b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow\n",
    "from tensorflow import keras\n",
    "import matplotlib.pyplot as plt\n",
    "from Bio import SeqIO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "949dec3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def from_files_to_vectors(fasta_path, motif='', motif_dictionary=sequence_dictionary_25, positive=True):\n",
    "    if positive:\n",
    "        proteins = list(SeqIO.parse(fasta_path+'adhesins.fasta', \"fasta\"))\n",
    "    else:\n",
    "        proteins = list(SeqIO.parse(fasta_path+'non_adhesins.fasta', \"fasta\"))\n",
    "    extension = \".out\"\n",
    "    files = [\"aac\", \"dpc\", \"ctdc\", \"ctdt\", \"ctdd\"]\n",
    "    if positive:\n",
    "        names = \"_pos\"\n",
    "    else:\n",
    "        names = \"_neg\"\n",
    "    for i in range(len(files)):\n",
    "        files[i] += names\n",
    "    datasets = [[] for el in files]\n",
    "    motif_datasets = [[] for el in files]\n",
    "    for i in range(len(files)):\n",
    "        with open(fasta_path+files[i]+extension) as f:\n",
    "            lines = f.readlines()[1:]\n",
    "            check_prot = 0\n",
    "            for line in lines:\n",
    "                information = line.split('\\t')\n",
    "                if not information[0] == proteins[check_prot].id:\n",
    "                    print(\"Error in protein order! Return\")\n",
    "                    return datasets\n",
    "                # place proteins with a given motif in a specific dataset\n",
    "                if positive:\n",
    "                    if information[0] in motif_dictionary[motif].keys():\n",
    "                        motif_datasets[i].append(np.array([float(el) for el in information[1:]]))\n",
    "                    else:\n",
    "                        datasets[i].append(np.array([float(el) for el in information[1:]]))\n",
    "                else:    \n",
    "                    datasets[i].append(np.array([float(el) for el in information[1:]]))\n",
    "                check_prot += 1\n",
    "        datasets[i] = np.array(datasets[i])\n",
    "        if positive:\n",
    "            motif_datasets[i] = np.array(motif_datasets[i])\n",
    "    if positive:\n",
    "        \n",
    "        return datasets, motif_datasets\n",
    "    return datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9fa451ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras import regularizers\n",
    "import pandas as pd\n",
    "\n",
    "class neural_network:\n",
    "    def __init__(self, K):\n",
    "        input = tensorflow.keras.Input(shape=(K,))\n",
    "        \n",
    "        dense = tensorflow.keras.layers.Dense(units=10, \n",
    "                                                activation='sigmoid',\n",
    "                                                kernel_regularizer=regularizers.L1L2(l1=1e-3, l2=1e-3),\n",
    "                                                bias_regularizer=regularizers.L2(1e-3),\n",
    "                                                activity_regularizer=regularizers.L2(1e-3))(input)\n",
    "        #norm = tensorflow.keras.layers.BatchNormalization()(dense)\n",
    "        #drop = tensorflow.keras.layers.Dropout(.1)(dense)\n",
    "        #dense = tensorflow.keras.layers.Dense(10, activation='sigmoid')(drop)\n",
    "        output = tensorflow.keras.layers.Dense(1, activation='sigmoid')(dense)\n",
    "\n",
    "        model = tensorflow.keras.models.Model(inputs=input, outputs=output)\n",
    "        model.compile(optimizer=tensorflow.keras.optimizers.Adam(learning_rate=0.0005), loss='binary_crossentropy', metrics='accuracy')\n",
    "        self.model = model\n",
    "\n",
    "\n",
    "def run_model_motifs(pos_datasets, neg_datasets, motif_datasets, projection_matrix='projection_matrix.npy',\\\n",
    "                    model_weights = 'ac_motif.h5', training_means = np.load('./mean.npy'), \\\n",
    "                    training_std_devs = np.load('./std.npy')):\n",
    "    \"\"\"Run multiple cross validations over different motifs to test model generalization\"\"\"\n",
    "    \n",
    "    # pre processing\n",
    "    y_pos = np.ones(pos_datasets[0].shape[0])\n",
    "    y_neg = np.zeros(neg_datasets[0].shape[0])\n",
    "    y_motif = np.ones(motif_datasets[0].shape[0])\n",
    "    \n",
    "    # attach datasets in order to obtain a matrix of (n, 20+400+39+39+195) features\n",
    "    rows = 0\n",
    "    n_pos = y_pos.shape[0]\n",
    "    n_neg = y_neg.shape[0]\n",
    "    rows = n_pos + n_neg\n",
    "    # feature vectors dimensions\n",
    "    columns = 0\n",
    "    for i in range(len(pos_datasets)):\n",
    "        tmp_dim = pos_datasets[i].shape[1]\n",
    "        columns += tmp_dim\n",
    "    # data matrix to process\n",
    "    X = np.zeros((rows, columns))\n",
    "    for i in range(n_pos):\n",
    "        X[i] = np.concatenate([pos_datasets[j][i] for j in range(5)])\n",
    "    for i in range(n_neg):\n",
    "        X[n_pos+i] = np.concatenate([neg_datasets[j][i] for j in range(5)])\n",
    "    # same for motif dataset\n",
    "    X_motifs = np.zeros((y_motif.shape[0], columns))\n",
    "    for i in range(y_motif.shape[0]):\n",
    "        X_motifs[i] = np.concatenate([motif_datasets[j][i] for j in range(5)])\n",
    "    \n",
    "    # permutation\n",
    "    np.random.seed(42)\n",
    "    y = np.concatenate((y_pos, y_neg), axis=0)\n",
    "    c = np.random.permutation(np.arange(y.shape[0]))\n",
    "    y = y[c]\n",
    "    X = X[c] \n",
    "    \n",
    "    # split datasets for training \n",
    "    X_train = X[:int(X.shape[0]*.5)]\n",
    "    X_val = X[int(X.shape[0]*.5):int(X.shape[0]*.75)]\n",
    "    X_test = X[int(X.shape[0]*.75):]\n",
    "\n",
    "    y_train = y[:int(y.shape[0]*.5)]\n",
    "    y_val = y[int(y.shape[0]*.5):int(y.shape[0]*.75)]\n",
    "    y_test = y[int(y.shape[0]*.75):]\n",
    "    \n",
    "    # standardize\n",
    "    def standardize(dataset):\n",
    "        \"\"\"Standardize dataset\"\"\"\n",
    "        std_dataset = np.zeros(dataset.shape)\n",
    "        for j in range(dataset.shape[1]):\n",
    "            column = dataset[:,j]\n",
    "            std_dataset[:,j] = (column - training_means[j]) / training_std_devs[j]\n",
    "        return std_dataset\n",
    "    \n",
    "    stdX = standardize(X_train)\n",
    "    stdX_val = standardize(X_val)    \n",
    "    stdX_test = standardize(X_test)    \n",
    "    stdX_motif = standardize(X_motifs)\n",
    "    \n",
    "    # project matrices\n",
    "    projection_matrix = np.load(projection_matrix)\n",
    "    X_train = stdX.dot(projection_matrix)\n",
    "    X_val = stdX_val.dot(projection_matrix)\n",
    "    X_test = stdX_test.dot(projection_matrix)\n",
    "    X_motif = stdX_motif.dot(projection_matrix)\n",
    "    \n",
    "    # initiate model\n",
    "    nn = neural_network(400)\n",
    "    # train\n",
    "    history = nn.model.fit(\n",
    "    x=X_train, \n",
    "    y=y_train,\n",
    "    batch_size=64,\n",
    "    epochs=1000,\n",
    "    verbose=0,\n",
    "    validation_data=(X_val, y_val),\n",
    "    shuffle=True,\n",
    "    callbacks=[tensorflow.keras.callbacks.EarlyStopping(\n",
    "    restore_best_weights=True,\n",
    "    patience=5\n",
    "        )])\n",
    "    \n",
    "    model_test = nn.model.evaluate(x=X_test, y=y_test)\n",
    "    print(model_test)\n",
    "    nn.model.save('ac_motif.h5')\n",
    "    model = tensorflow.keras.models.load_model(model_weights)\n",
    "    motif_test = model.evaluate(x=X_motif, y=y_motif)\n",
    "    print(motif_test)\n",
    "    return model_test, motif_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ee079c3f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-11-24 13:26:47.913347: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory\n",
      "2022-11-24 13:26:47.913490: W tensorflow/stream_executor/cuda/cuda_driver.cc:263] failed call to cuInit: UNKNOWN ERROR (303)\n",
      "2022-11-24 13:26:47.913520: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (ad383b76f2b6): /proc/driver/nvidia/version does not exist\n",
      "2022-11-24 13:26:47.913836: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F AVX512_VNNI FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "58/58 [==============================] - 0s 1ms/step - loss: 0.2316 - accuracy: 0.9369\n",
      "[0.23156243562698364, 0.936859130859375]\n",
      "1/1 [==============================] - 0s 179ms/step - loss: 0.4061 - accuracy: 0.9333\n",
      "[0.40609607100486755, 0.9333333373069763]\n",
      "57/57 [==============================] - 0s 1ms/step - loss: 0.2371 - accuracy: 0.9312\n",
      "[0.237071692943573, 0.9312431216239929]\n",
      "5/5 [==============================] - 0s 2ms/step - loss: 0.1551 - accuracy: 0.9673\n",
      "[0.15505166351795197, 0.9673202633857727]\n",
      "58/58 [==============================] - 0s 1ms/step - loss: 0.2085 - accuracy: 0.9429\n",
      "[0.2085108608007431, 0.9428879022598267]\n",
      "1/1 [==============================] - 0s 176ms/step - loss: 0.0664 - accuracy: 1.0000\n",
      "[0.06643704324960709, 1.0]\n",
      "58/58 [==============================] - 0s 826us/step - loss: 0.2162 - accuracy: 0.9445\n",
      "[0.21620430052280426, 0.9444743990898132]\n",
      "1/1 [==============================] - 0s 104ms/step - loss: 1.1477 - accuracy: 0.4000\n",
      "[1.1476943492889404, 0.4000000059604645]\n",
      "58/58 [==============================] - 0s 1ms/step - loss: 0.2172 - accuracy: 0.9397\n",
      "[0.21715305745601654, 0.9396551847457886]\n",
      "1/1 [==============================] - 0s 140ms/step - loss: 0.0668 - accuracy: 1.0000\n",
      "[0.06679003685712814, 1.0]\n",
      "45/45 [==============================] - 0s 840us/step - loss: 0.1979 - accuracy: 0.9467\n",
      "[0.19793277978897095, 0.9467414021492004]\n",
      "54/54 [==============================] - 0s 842us/step - loss: 1.7432 - accuracy: 0.3915\n",
      "[1.7431813478469849, 0.39150670170783997]\n",
      "57/57 [==============================] - 0s 1ms/step - loss: 0.2114 - accuracy: 0.9450\n",
      "[0.21139000356197357, 0.9449642300605774]\n",
      "5/5 [==============================] - 0s 2ms/step - loss: 0.5380 - accuracy: 0.8418\n",
      "[0.5379708409309387, 0.8417721390724182]\n",
      "57/57 [==============================] - 0s 1ms/step - loss: 0.2399 - accuracy: 0.9308\n",
      "[0.2398894876241684, 0.9308452010154724]\n",
      "5/5 [==============================] - 0s 2ms/step - loss: 0.2062 - accuracy: 0.9559\n",
      "[0.20623670518398285, 0.9558823704719543]\n",
      "58/58 [==============================] - 0s 1ms/step - loss: 0.2178 - accuracy: 0.9397\n",
      "[0.21777014434337616, 0.9396551847457886]\n",
      "1/1 [==============================] - 0s 180ms/step - loss: 0.2276 - accuracy: 1.0000\n",
      "[0.227582648396492, 1.0]\n",
      "58/58 [==============================] - 0s 1ms/step - loss: 0.2194 - accuracy: 0.9391\n",
      "[0.21938979625701904, 0.9390835762023926]\n",
      "1/1 [==============================] - 0s 180ms/step - loss: 1.5721 - accuracy: 0.2500\n",
      "[1.5721489191055298, 0.25]\n",
      "58/58 [==============================] - 0s 2ms/step - loss: 0.2145 - accuracy: 0.9407\n",
      "[0.21449029445648193, 0.9407327771186829]\n",
      "1/1 [==============================] - 0s 140ms/step - loss: 0.2284 - accuracy: 1.0000\n",
      "[0.22838757932186127, 1.0]\n",
      "58/58 [==============================] - 0s 750us/step - loss: 0.2161 - accuracy: 0.9345\n",
      "[0.2160601168870926, 0.9344883561134338]\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.0948 - accuracy: 1.0000\n",
      "[0.0948077142238617, 1.0]\n",
      "55/55 [==============================] - 0s 1ms/step - loss: 0.2289 - accuracy: 0.9333\n",
      "[0.22889205813407898, 0.9332572817802429]\n",
      "13/13 [==============================] - 0s 2ms/step - loss: 0.5727 - accuracy: 0.7899\n",
      "[0.5726921558380127, 0.7898550629615784]\n",
      "57/57 [==============================] - 0s 1ms/step - loss: 0.2365 - accuracy: 0.9315\n",
      "[0.23649370670318604, 0.9314693212509155]\n",
      "5/5 [==============================] - 0s 2ms/step - loss: 0.4098 - accuracy: 0.8760\n",
      "[0.4097706377506256, 0.8759689927101135]\n",
      "58/58 [==============================] - 0s 1ms/step - loss: 0.2260 - accuracy: 0.9358\n",
      "[0.22596533596515656, 0.9357648491859436]\n",
      "3/3 [==============================] - 0s 3ms/step - loss: 0.1607 - accuracy: 0.9605\n",
      "[0.1607462763786316, 0.9605262875556946]\n",
      "58/58 [==============================] - 0s 946us/step - loss: 0.2155 - accuracy: 0.9407\n",
      "[0.21552985906600952, 0.9407327771186829]\n",
      "1/1 [==============================] - 0s 114ms/step - loss: 0.1794 - accuracy: 1.0000\n",
      "[0.17936402559280396, 1.0]\n",
      "58/58 [==============================] - 0s 1ms/step - loss: 0.2169 - accuracy: 0.9375\n",
      "[0.21685998141765594, 0.9374663233757019]\n",
      "1/1 [==============================] - 0s 175ms/step - loss: 0.7679 - accuracy: 0.6000\n",
      "[0.7678630352020264, 0.6000000238418579]\n",
      "58/58 [==============================] - 0s 1ms/step - loss: 0.2205 - accuracy: 0.9390\n",
      "[0.22045530378818512, 0.9389519095420837]\n",
      "1/1 [==============================] - 0s 173ms/step - loss: 1.0728 - accuracy: 0.6364\n",
      "[1.0728435516357422, 0.6363636255264282]\n",
      "58/58 [==============================] - 0s 872us/step - loss: 0.2401 - accuracy: 0.9288\n",
      "[0.2400583028793335, 0.9288026094436646]\n",
      "1/1 [==============================] - 0s 116ms/step - loss: 4.5264 - accuracy: 0.0000e+00\n",
      "[4.526416301727295, 0.0]\n",
      "58/58 [==============================] - 0s 1ms/step - loss: 0.2105 - accuracy: 0.9407\n",
      "[0.21050213277339935, 0.9407327771186829]\n",
      "1/1 [==============================] - 0s 122ms/step - loss: 0.0687 - accuracy: 1.0000\n",
      "[0.06870675086975098, 1.0]\n",
      "57/57 [==============================] - 0s 1ms/step - loss: 0.2395 - accuracy: 0.9310\n",
      "[0.23953743278980255, 0.931034505367279]\n",
      "8/8 [==============================] - 0s 2ms/step - loss: 0.1495 - accuracy: 0.9786\n",
      "[0.149495929479599, 0.9786324501037598]\n",
      "58/58 [==============================] - 0s 748us/step - loss: 0.2194 - accuracy: 0.9399\n",
      "[0.21935375034809113, 0.9399350881576538]\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2468 - accuracy: 0.9091\n",
      "[0.2468455582857132, 0.9090909361839294]\n",
      "58/58 [==============================] - 0s 2ms/step - loss: 0.2149 - accuracy: 0.9407\n",
      "[0.2149307280778885, 0.9407327771186829]\n",
      "1/1 [==============================] - 0s 180ms/step - loss: 0.1433 - accuracy: 1.0000\n",
      "[0.1433204561471939, 1.0]\n",
      "58/58 [==============================] - 0s 729us/step - loss: 0.2391 - accuracy: 0.9294\n",
      "[0.23905591666698456, 0.9293800592422485]\n",
      "1/1 [==============================] - 0s 106ms/step - loss: 0.0910 - accuracy: 1.0000\n",
      "[0.09103281050920486, 1.0]\n",
      "54/54 [==============================] - 0s 1ms/step - loss: 0.1943 - accuracy: 0.9463\n",
      "[0.19428986310958862, 0.9463244080543518]\n",
      "18/18 [==============================] - 0s 1ms/step - loss: 0.7246 - accuracy: 0.7540\n",
      "[0.724646806716919, 0.753954291343689]\n"
     ]
    }
   ],
   "source": [
    "outlist = []\n",
    "for motif in sequence_dictionary_25:\n",
    "    pos_datasets, motif_datasets = from_files_to_vectors(\"./data/25_similarity/positive/\", motif, positive=True)\n",
    "    neg_datasets = from_files_to_vectors(\"./data/25_similarity/negative/\", positive=False)\n",
    "    #print(motif, sum(sequence_dictionary_25[motif].values()))\n",
    "    model_test, motif_test = run_model_motifs(pos_datasets, neg_datasets, motif_datasets)\n",
    "    outlist.append([motif, sum(sequence_dictionary_25[motif].values()), model_test[1],\\\n",
    "                    motif_test[1]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a4fa3962",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The mean accuracy on unseen adhesin motifs is 0.8097682595252991\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>motif</th>\n",
       "      <th>proteins</th>\n",
       "      <th>model_accuracy</th>\n",
       "      <th>model_accuracy_on_motif</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>PF08363</td>\n",
       "      <td>15</td>\n",
       "      <td>0.936859</td>\n",
       "      <td>0.933333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>PF05737</td>\n",
       "      <td>153</td>\n",
       "      <td>0.931243</td>\n",
       "      <td>0.967320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>PF08829</td>\n",
       "      <td>2</td>\n",
       "      <td>0.942888</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>PF18220</td>\n",
       "      <td>5</td>\n",
       "      <td>0.944474</td>\n",
       "      <td>0.400000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>PF17480</td>\n",
       "      <td>1</td>\n",
       "      <td>0.939655</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>PF12799</td>\n",
       "      <td>1719</td>\n",
       "      <td>0.946741</td>\n",
       "      <td>0.391507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>PF18573</td>\n",
       "      <td>158</td>\n",
       "      <td>0.944964</td>\n",
       "      <td>0.841772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>PF03212</td>\n",
       "      <td>136</td>\n",
       "      <td>0.930845</td>\n",
       "      <td>0.955882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>PF07979</td>\n",
       "      <td>1</td>\n",
       "      <td>0.939655</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>PF02216</td>\n",
       "      <td>4</td>\n",
       "      <td>0.939084</td>\n",
       "      <td>0.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>PF15403</td>\n",
       "      <td>1</td>\n",
       "      <td>0.940733</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>PF18651</td>\n",
       "      <td>37</td>\n",
       "      <td>0.934488</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>PF05658</td>\n",
       "      <td>414</td>\n",
       "      <td>0.933257</td>\n",
       "      <td>0.789855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>PF08341</td>\n",
       "      <td>129</td>\n",
       "      <td>0.931469</td>\n",
       "      <td>0.875969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>PF07675</td>\n",
       "      <td>76</td>\n",
       "      <td>0.935765</td>\n",
       "      <td>0.960526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>PF18304</td>\n",
       "      <td>1</td>\n",
       "      <td>0.940733</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>PF09160</td>\n",
       "      <td>5</td>\n",
       "      <td>0.937466</td>\n",
       "      <td>0.600000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>PF18652</td>\n",
       "      <td>22</td>\n",
       "      <td>0.938952</td>\n",
       "      <td>0.636364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>PF09403</td>\n",
       "      <td>10</td>\n",
       "      <td>0.928803</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>PF18873</td>\n",
       "      <td>2</td>\n",
       "      <td>0.940733</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>PF18483</td>\n",
       "      <td>234</td>\n",
       "      <td>0.931035</td>\n",
       "      <td>0.978632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>PF10425</td>\n",
       "      <td>33</td>\n",
       "      <td>0.939935</td>\n",
       "      <td>0.909091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>PF11966</td>\n",
       "      <td>1</td>\n",
       "      <td>0.940733</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>PF15401</td>\n",
       "      <td>5</td>\n",
       "      <td>0.929380</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>PF07691</td>\n",
       "      <td>569</td>\n",
       "      <td>0.946324</td>\n",
       "      <td>0.753954</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      motif  proteins  model_accuracy  model_accuracy_on_motif\n",
       "0   PF08363        15        0.936859                 0.933333\n",
       "1   PF05737       153        0.931243                 0.967320\n",
       "2   PF08829         2        0.942888                 1.000000\n",
       "3   PF18220         5        0.944474                 0.400000\n",
       "4   PF17480         1        0.939655                 1.000000\n",
       "5   PF12799      1719        0.946741                 0.391507\n",
       "6   PF18573       158        0.944964                 0.841772\n",
       "7   PF03212       136        0.930845                 0.955882\n",
       "8   PF07979         1        0.939655                 1.000000\n",
       "9   PF02216         4        0.939084                 0.250000\n",
       "10  PF15403         1        0.940733                 1.000000\n",
       "11  PF18651        37        0.934488                 1.000000\n",
       "12  PF05658       414        0.933257                 0.789855\n",
       "13  PF08341       129        0.931469                 0.875969\n",
       "14  PF07675        76        0.935765                 0.960526\n",
       "15  PF18304         1        0.940733                 1.000000\n",
       "16  PF09160         5        0.937466                 0.600000\n",
       "17  PF18652        22        0.938952                 0.636364\n",
       "18  PF09403        10        0.928803                 0.000000\n",
       "19  PF18873         2        0.940733                 1.000000\n",
       "20  PF18483       234        0.931035                 0.978632\n",
       "21  PF10425        33        0.939935                 0.909091\n",
       "22  PF11966         1        0.940733                 1.000000\n",
       "23  PF15401         5        0.929380                 1.000000\n",
       "24  PF07691       569        0.946324                 0.753954"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(outlist, columns = ['motif', 'proteins', 'model_accuracy', 'model_accuracy_on_motif'])\n",
    "print(f'The mean accuracy on unseen adhesin motifs is {df[\"model_accuracy_on_motif\"].mean()}')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "3b8e2168",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot: >"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAd90lEQVR4nO3dfZBV9X348c/usnthnV0UKbCbrBFtUxIwMpHAKEmDLcgQgzrTRjtYS22radw0VWaMWIMs4lNsxmFqqDbmwXQqkj6IddSgWyJhiBiVh46/alAqSUwsWJK4i2y9XnfP7w+GpeuuyJJzvuSS12vm/nHPnv2e7364sm/vvcvWZFmWBQBAIrVHewMAwK8X8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmNONobeLu+vr545ZVXoqmpKWpqao72dgCAw5BlWezduzdaW1ujtvbQz238ysXHK6+8Em1tbUd7GwDAEXj55Zfjve997yHP+ZWLj6ampojYv/nm5uZc165UKvHYY4/FOeecE/X19bmuzUHmnIY5p2HO6Zh1GkXNubu7O9ra2vq/jx/Kr1x8HHippbm5uZD4aGxsjObmZg/sAplzGuachjmnY9ZpFD3nw3nLhDecAgBJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSGnG0NwAADHby4ocLWbdUl8Vt0wtZ+rB55gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJDUsONjw4YNMX/+/GhtbY2ampp44IEH+j9WqVTimmuuidNOOy2OO+64aG1tjT/+4z+OV155Jc89AwBVbNjxsW/fvjj99NNj5cqVgz7W09MTW7ZsiSVLlsSWLVvi/vvvj+3bt8d5552Xy2YBgOo3YrifMG/evJg3b96QHxs9enR0dnYOOPblL385pk+fHj/+8Y/jpJNOOrJdAgDHjGHHx3B1dXVFTU1NHH/88UN+vFwuR7lc7r/f3d0dEftfwqlUKrnu5cB6ea/LQOachjmnYc7pmPVApbqsmHVr969b1PfYw1GTZdkRf3U1NTWxZs2auOCCC4b8+BtvvBEzZ86MSZMmxb333jvkOR0dHbFs2bJBx1etWhWNjY1HujUAIKGenp5YsGBBdHV1RXNz8yHPLSw+KpVK/P7v/3785Cc/ifXr17/jRoZ65qOtrS327NnzrpsfrkqlEp2dnTFnzpyor6/PdW0OMuc0zDkNc07HrAea0vFoIeuWarNYPq0v9zl3d3fH2LFjDys+CnnZpVKpxIUXXhg/+tGP4jvf+c4hN1EqlaJUKg06Xl9fX9iDr8i1Ocic0zDnNMw5HbPer9xbU+j6ec95OGvlHh8HwuPFF1+Mxx9/PE488cS8LwEAVLFhx8frr78eO3bs6L+/c+fO2LZtW4wZMyZaWlriD/7gD2LLli3x0EMPRW9vb+zatSsiIsaMGRMNDQ357RwAqErDjo9nnnkmzj777P77ixYtioiIhQsXRkdHRzz44IMRETF16tQBn/f444/HrFmzjnynAMAxYdjxMWvWrDjUe1R/ifevAgC/BvxuFwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkhh0fGzZsiPnz50dra2vU1NTEAw88MODjWZbF9ddfHy0tLTFq1KiYPXt2vPjii3ntFwCocsOOj3379sXpp58eK1euHPLjt912W/zt3/5t3HXXXfH9738/jjvuuJg7d2688cYbv/RmAYDqN2K4nzBv3ryYN2/ekB/LsixWrFgRX/jCF+L888+PiIh/+Id/iPHjx8cDDzwQf/iHf/jL7RYAqHq5vudj586dsWvXrpg9e3b/sdGjR8eMGTNi06ZNeV4KAKhSw37m41B27doVERHjx48fcHz8+PH9H3u7crkc5XK5/353d3dERFQqlahUKnlur3+9vNdlIHNOw5zTMOd0zHqgUl1WzLq1+9ct6nvs4cg1Po7ELbfcEsuWLRt0/LHHHovGxsZCrtnZ2VnIugxkzmmYcxrmnI5Z73fb9GLXz3vOPT09h31urvExYcKEiIjYvXt3tLS09B/fvXt3TJ06dcjPufbaa2PRokX997u7u6OtrS3OOeecaG5uznN7UalUorOzM+bMmRP19fW5rs1B5pyGOadhzumY9UBTOh4tZN1SbRbLp/XlPucDr1wcjlzjY+LEiTFhwoRYt25df2x0d3fH97///fjMZz4z5OeUSqUolUqDjtfX1xf24CtybQ4y5zTMOQ1zTses9yv31hS6ft5zHs5aw46P119/PXbs2NF/f+fOnbFt27YYM2ZMnHTSSXHllVfGjTfeGL/1W78VEydOjCVLlkRra2tccMEFw70UAHAMGnZ8PPPMM3H22Wf33z/wksnChQvjnnvuic9//vOxb9++uPzyy+O1116Lj370o7F27doYOXJkfrsGAKrWsONj1qxZkWXv/A7cmpqauOGGG+KGG274pTYGAByb/G4XACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKRyj4/e3t5YsmRJTJw4MUaNGhWnnnpqLF++PLIsy/tSAEAVGpH3gl/84hfjzjvvjG9+85sxefLkeOaZZ+LSSy+N0aNHx+c+97m8LwcAVJnc4+OJJ56I888/P84999yIiDj55JPjvvvui6eeeirvSwEAVSj3l13OOuusWLduXbzwwgsREfEf//EfsXHjxpg3b17elwIAqlDuz3wsXrw4uru7Y9KkSVFXVxe9vb1x0003xcUXXzzk+eVyOcrlcv/97u7uiIioVCpRqVRy3duB9fJel4HMOQ1zTsOc0zHrgUp1xbxXslS7f92ivscejpos53eCrl69Oq6++ur4m7/5m5g8eXJs27Ytrrzyyrj99ttj4cKFg87v6OiIZcuWDTq+atWqaGxszHNrAEBBenp6YsGCBdHV1RXNzc2HPDf3+Ghra4vFixdHe3t7/7Ebb7wx/vEf/zF+8IMfDDp/qGc+2traYs+ePe+6+eGqVCrR2dkZc+bMifr6+lzX5iBzTsOc0zDndMx6oCkdjxaybqk2i+XT+nKfc3d3d4wdO/aw4iP3l116enqitnbgW0nq6uqir69vyPNLpVKUSqVBx+vr6wt78BW5NgeZcxrmnIY5p2PW+5V7awpdP+85D2et3ONj/vz5cdNNN8VJJ50UkydPjq1bt8btt98ef/qnf5r3pQCAKpR7fNxxxx2xZMmSuOKKK+LVV1+N1tbW+PSnPx3XX3993pcCAKpQ7vHR1NQUK1asiBUrVuS9NABwDPC7XQCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFKFxMdPf/rT+KM/+qM48cQTY9SoUXHaaafFM888U8SlAIAqMyLvBX/xi1/EzJkz4+yzz45vf/vb8Ru/8Rvx4osvxgknnJD3pQCAKpR7fHzxi1+Mtra2+MY3vtF/bOLEiXlfBgCoUrnHx4MPPhhz586NT33qU/Hd73433vOe98QVV1wRl1122ZDnl8vlKJfL/fe7u7sjIqJSqUSlUsl1bwfWy3tdBjLnNMw5DXNOx6wHKtVlxaxbu3/dor7HHo6aLMty/epGjhwZERGLFi2KT33qU/H000/HX/3VX8Vdd90VCxcuHHR+R0dHLFu2bNDxVatWRWNjY55bAwAK0tPTEwsWLIiurq5obm4+5Lm5x0dDQ0NMmzYtnnjiif5jn/vc5+Lpp5+OTZs2DTp/qGc+2traYs+ePe+6+eGqVCrR2dkZc+bMifr6+lzX5iBzTsOc0zDndMx6oCkdjxaybqk2i+XT+nKfc3d3d4wdO/aw4iP3l11aWlrigx/84IBjH/jAB+Jf//Vfhzy/VCpFqVQadLy+vr6wB1+Ra3OQOadhzmmYczpmvV+5t6bQ9fOe83DWyv1HbWfOnBnbt28fcOyFF16I973vfXlfCgCoQrnHx1VXXRVPPvlk3HzzzbFjx45YtWpVfOUrX4n29va8LwUAVKHc4+MjH/lIrFmzJu67776YMmVKLF++PFasWBEXX3xx3pcCAKpQ7u/5iIj45Cc/GZ/85CeLWBoAqHJ+twsAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUoXHx6233ho1NTVx5ZVXFn0pAKAKFBofTz/9dPz93/99fOhDHyryMgBAFSksPl5//fW4+OKL4+67744TTjihqMsAAFVmRFELt7e3x7nnnhuzZ8+OG2+88R3PK5fLUS6X++93d3dHRESlUolKpZLrng6sl/e6DGTOaZhzGuacjlkPVKrLilm3dv+6RX2PPRw1WZbl/tWtXr06brrppnj66adj5MiRMWvWrJg6dWqsWLFi0LkdHR2xbNmyQcdXrVoVjY2NeW8NAChAT09PLFiwILq6uqK5ufmQ5+YeHy+//HJMmzYtOjs7+9/rcaj4GOqZj7a2ttizZ8+7bn64KpVKdHZ2xpw5c6K+vj7XtTnInNMw5zTMOR2zHmhKx6OFrFuqzWL5tL7c59zd3R1jx449rPjI/WWXzZs3x6uvvhof/vCH+4/19vbGhg0b4stf/nKUy+Woq6vr/1ipVIpSqTRonfr6+sIefEWuzUHmnIY5p2HO6Zj1fuXemkLXz3vOw1kr9/j4vd/7vXj22WcHHLv00ktj0qRJcc011wwIDwDg10/u8dHU1BRTpkwZcOy4446LE088cdBxAODXj3/hFABIqrAftf2/1q9fn+IyAEAV8MwHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFK5x8ctt9wSH/nIR6KpqSnGjRsXF1xwQWzfvj3vywAAVSr3+Pjud78b7e3t8eSTT0ZnZ2dUKpU455xzYt++fXlfCgCoQiPyXnDt2rUD7t9zzz0xbty42Lx5c/zO7/xO3pcDAKpM7vHxdl1dXRERMWbMmCE/Xi6Xo1wu99/v7u6OiIhKpRKVSiXXvRxYL+91Gcic0zDnNMw5HbMeqFSXFbNu7f51i/oeezhqsiwr5quLiL6+vjjvvPPitddei40bNw55TkdHRyxbtmzQ8VWrVkVjY2NRWwMActTT0xMLFiyIrq6uaG5uPuS5hcbHZz7zmfj2t78dGzdujPe+971DnjPUMx9tbW2xZ8+ed938cFUqlejs7Iw5c+ZEfX19rmtzkDmnYc5pHJjzkmdqo9xXk/v6/69jbu5rFm1Kx6OFrFuqzWL5tL6qe0wXNY+iFDXn7u7uGDt27GHFR2Evu3z2s5+Nhx56KDZs2PCO4RERUSqVolQqDTpeX19f2IOvyLU5yJzTMOc0yn01Ue7NPz6q8c+uiDn8X9X2mC56HkXJe87DWSv3+MiyLP7yL/8y1qxZE+vXr4+JEyfmfQkAoIrlHh/t7e2xatWq+Ld/+7doamqKXbt2RUTE6NGjY9SoUXlfDgCoMrn/Ox933nlndHV1xaxZs6KlpaX/9q1vfSvvSwEAVaiQl10AAN6J3+0CACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSGnG0N3A0TOl4NMq9Nbmu+cNbz811Pd7ZyYsfLmTdIv8Mi9hzqS6L26YX83iO8Jjm6CnqMc2vDs98AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkiosPlauXBknn3xyjBw5MmbMmBFPPfVUUZcCAKpIIfHxrW99KxYtWhRLly6NLVu2xOmnnx5z586NV199tYjLAQBVpJD4uP322+Oyyy6LSy+9ND74wQ/GXXfdFY2NjfH1r3+9iMsBAFVkRN4Lvvnmm7F58+a49tpr+4/V1tbG7NmzY9OmTYPOL5fLUS6X++93dXVFRMTPf/7zqFQque6tUqlET09PjKjURm9fTa5r/+xnP8t1vWp2YM4/+9nPor6+Pvf1R7y1L/c1I4r9MyxizyP6sujp6Svk8RzhMX1AkX9vRFTnnIv6b7DoxzT7HZhz3n9H7927NyIisix795OznP30pz/NIiJ74oknBhy/+uqrs+nTpw86f+nSpVlEuLm5ubm5uR0Dt5dffvldWyH3Zz6G69prr41Fixb13+/r64uf//znceKJJ0ZNTb7l293dHW1tbfHyyy9Hc3NzrmtzkDmnYc5pmHM6Zp1GUXPOsiz27t0bra2t73pu7vExduzYqKuri927dw84vnv37pgwYcKg80ulUpRKpQHHjj/++Ly3NUBzc7MHdgLmnIY5p2HO6Zh1GkXMefTo0Yd1Xu5vOG1oaIgzzjgj1q1b13+sr68v1q1bF2eeeWbelwMAqkwhL7ssWrQoFi5cGNOmTYvp06fHihUrYt++fXHppZcWcTkAoIoUEh8XXXRR/M///E9cf/31sWvXrpg6dWqsXbs2xo8fX8TlDlupVIqlS5cOepmHfJlzGuachjmnY9Zp/CrMuSbLDudnYgAA8uF3uwAASYkPACAp8QEAJCU+AICkjrn4WLlyZZx88skxcuTImDFjRjz11FOHPP+f//mfY9KkSTFy5Mg47bTT4pFHHkm00+o2nDnffffd8bGPfSxOOOGEOOGEE2L27Nnv+ufCfsN9PB+wevXqqKmpiQsuuKDYDR4jhjvn1157Ldrb26OlpSVKpVK8//3v93fHYRjunFesWBG//du/HaNGjYq2tra46qqr4o033ki02+q0YcOGmD9/frS2tkZNTU088MAD7/o569evjw9/+MNRKpXiN3/zN+Oee+4pfJ+5/26Xo2n16tVZQ0ND9vWvfz37z//8z+yyyy7Ljj/++Gz37t1Dnv+9730vq6ury2677bbsueeey77whS9k9fX12bPPPpt459VluHNesGBBtnLlymzr1q3Z888/n/3Jn/xJNnr06OwnP/lJ4p1Xl+HO+YCdO3dm73nPe7KPfexj2fnnn59ms1VsuHMul8vZtGnTsk984hPZxo0bs507d2br16/Ptm3blnjn1WW4c7733nuzUqmU3XvvvdnOnTuzRx99NGtpacmuuuqqxDuvLo888kh23XXXZffff38WEdmaNWsOef5LL72UNTY2ZosWLcqee+657I477sjq6uqytWvXFrrPYyo+pk+fnrW3t/ff7+3tzVpbW7NbbrllyPMvvPDC7Nxzzx1wbMaMGdmnP/3pQvdZ7YY757d76623sqampuyb3/xmUVs8JhzJnN96663srLPOyr761a9mCxcuFB+HYbhzvvPOO7NTTjkle/PNN1Nt8Zgw3Dm3t7dnv/u7vzvg2KJFi7KZM2cWus9jyeHEx+c///ls8uTJA45ddNFF2dy5cwvcWZYdMy+7vPnmm7F58+aYPXt2/7Ha2tqYPXt2bNq0acjP2bRp04DzIyLmzp37judzZHN+u56enqhUKjFmzJiitln1jnTON9xwQ4wbNy7+7M/+LMU2q96RzPnBBx+MM888M9rb22P8+PExZcqUuPnmm6O3tzfVtqvOkcz5rLPOis2bN/e/NPPSSy/FI488Ep/4xCeS7PnXxdH6PnjUf6ttXvbs2RO9vb2D/hXV8ePHxw9+8IMhP2fXrl1Dnr9r167C9lntjmTOb3fNNddEa2vroAc8Bx3JnDdu3Bhf+9rXYtu2bQl2eGw4kjm/9NJL8Z3vfCcuvvjieOSRR2LHjh1xxRVXRKVSiaVLl6bYdtU5kjkvWLAg9uzZEx/96Ecjy7J466234i/+4i/ir//6r1Ns+dfGO30f7O7ujv/93/+NUaNGFXLdY+aZD6rDrbfeGqtXr441a9bEyJEjj/Z2jhl79+6NSy65JO6+++4YO3bs0d7OMa2vry/GjRsXX/nKV+KMM86Iiy66KK677rq46667jvbWjinr16+Pm2++Of7u7/4utmzZEvfff388/PDDsXz58qO9NXJwzDzzMXbs2Kirq4vdu3cPOL579+6YMGHCkJ8zYcKEYZ3Pkc35gC996Utx6623xr//+7/Hhz70oSK3WfWGO+f/+q//ih/+8Icxf/78/mN9fX0RETFixIjYvn17nHrqqcVuugodyeO5paUl6uvro66urv/YBz7wgdi1a1e8+eab0dDQUOieq9GRzHnJkiVxySWXxJ//+Z9HRMRpp50W+/bti8svvzyuu+66qK31/855eKfvg83NzYU96xFxDD3z0dDQEGeccUasW7eu/1hfX1+sW7cuzjzzzCE/58wzzxxwfkREZ2fnO57Pkc05IuK2226L5cuXx9q1a2PatGkptlrVhjvnSZMmxbPPPhvbtm3rv5133nlx9tlnx7Zt26KtrS3l9qvGkTyeZ86cGTt27OiPu4iIF154IVpaWoTHOziSOff09AwKjAPBl/mVZLk5at8HC307a2KrV6/OSqVSds8992TPPfdcdvnll2fHH398tmvXrizLsuySSy7JFi9e3H/+9773vWzEiBHZl770pez555/Pli5d6kdtD8Nw53zrrbdmDQ0N2b/8y79k//3f/91/27t379H6EqrCcOf8dn7a5fAMd84//vGPs6ampuyzn/1stn379uyhhx7Kxo0bl914441H60uoCsOd89KlS7Ompqbsvvvuy1566aXssccey0499dTswgsvPFpfQlXYu3dvtnXr1mzr1q1ZRGS33357tnXr1uxHP/pRlmVZtnjx4uySSy7pP//Aj9peffXV2fPPP5+tXLnSj9oeiTvuuCM76aSTsoaGhmz69OnZk08+2f+xj3/849nChQsHnP9P//RP2fvf//6soaEhmzx5cvbwww8n3nF1Gs6c3/e+92URMei2dOnS9BuvMsN9PP9f4uPwDXfOTzzxRDZjxoysVCplp5xySnbTTTdlb731VuJdV5/hzLlSqWQdHR3Zqaeemo0cOTJra2vLrrjiiuwXv/hF+o1Xkccff3zIv28PzHbhwoXZxz/+8UGfM3Xq1KyhoSE75ZRTsm984xuF77Mmyzx/BQCkc8y85wMAqA7iAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AIKn/D38/z5NLkw9lAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df['model_accuracy_on_motif'].hist(bins=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ecfcbd2e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}