1_Benchmarking_CARLA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import importlib\n",
    "\n",
    "import pandas as pd\n",
    "from IPython.display import display\n",
    "\n",
    "import warnings\n",
    "\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "#from carla.data.causal_model import CausalModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Using Python-MIP package version 1.12.0 [model.py <module>]\n"
     ]
    }
   ],
   "source": [
    "import Semantic_Meaningfulness \n",
    "import carla\n",
    "carla.data.causal_model=Semantic_Meaningfulness.carla_adaptions.causal_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import torch\n",
    "import random\n",
    "\n",
    "#SEED Setting\n",
    "seed = 42\n",
    "np.random.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "random.seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'Semantic_Meaningfulness.carla_adaptions.causal_model.CausalModel'>\n",
      "3\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-3.822251</td>\n",
       "      <td>3.206103</td>\n",
       "      <td>1.394095</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.125186</td>\n",
       "      <td>-0.267489</td>\n",
       "      <td>-0.065593</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.388819</td>\n",
       "      <td>-2.420645</td>\n",
       "      <td>0.119093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.173873</td>\n",
       "      <td>-0.715379</td>\n",
       "      <td>-1.201855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.637307</td>\n",
       "      <td>1.071684</td>\n",
       "      <td>0.379442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.971876</td>\n",
       "      <td>-0.067054</td>\n",
       "      <td>-1.317657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.792395</td>\n",
       "      <td>-2.272653</td>\n",
       "      <td>0.620624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.994518</td>\n",
       "      <td>-3.212899</td>\n",
       "      <td>-0.900646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-2.854051</td>\n",
       "      <td>3.213437</td>\n",
       "      <td>-0.361817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-2.368446</td>\n",
       "      <td>1.890917</td>\n",
       "      <td>-0.205989</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      label        x1        x2        x3\n",
       "0       1.0 -3.822251  3.206103  1.394095\n",
       "1       0.0 -0.125186 -0.267489 -0.065593\n",
       "2       0.0  1.388819 -2.420645  0.119093\n",
       "3       0.0 -0.173873 -0.715379 -1.201855\n",
       "4       1.0 -0.637307  1.071684  0.379442\n",
       "...     ...       ...       ...       ...\n",
       "9995    0.0  0.971876 -0.067054 -1.317657\n",
       "9996    1.0  1.792395 -2.272653  0.620624\n",
       "9997    0.0  1.994518 -3.212899 -0.900646\n",
       "9998    0.0 -2.854051  3.213437 -0.361817\n",
       "9999    0.0 -2.368446  1.890917 -0.205989\n",
       "\n",
       "[10000 rows x 4 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "# generate data\n",
    "scm = carla.data.causal_model.CausalModel(\"sanity-3-lin\")\n",
    "print(type(scm))\n",
    "dataset = scm.generate_dataset(10000, False)\n",
    "\n",
    "display(dataset.df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.43.0 (0)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"90pt\" height=\"188pt\"\n",
       " viewBox=\"0.00 0.00 90.00 188.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 184)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-184 86,-184 86,4 -4,4\"/>\n",
       "<!-- x3 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>x3</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"27\" cy=\"-18\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\">x3</text>\n",
       "</g>\n",
       "<!-- x1 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>x1</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"27\" cy=\"-162\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-158.3\" font-family=\"Times,serif\" font-size=\"14.00\">x1</text>\n",
       "</g>\n",
       "<!-- x1&#45;&gt;x3 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>x1&#45;&gt;x3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M23.75,-143.89C21.95,-133.54 19.91,-120.06 19,-108 17.8,-92.04 17.8,-87.96 19,-72 19.64,-63.52 20.84,-54.34 22.12,-46.04\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"25.58,-46.55 23.75,-36.11 18.68,-45.41 25.58,-46.55\"/>\n",
       "</g>\n",
       "<!-- x2 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>x2</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"55\" cy=\"-90\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"55\" y=\"-86.3\" font-family=\"Times,serif\" font-size=\"14.00\">x2</text>\n",
       "</g>\n",
       "<!-- x1&#45;&gt;x2 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>x1&#45;&gt;x2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M33.64,-144.41C36.91,-136.22 40.94,-126.14 44.62,-116.95\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47.95,-118.05 48.41,-107.47 41.45,-115.45 47.95,-118.05\"/>\n",
       "</g>\n",
       "<!-- x2&#45;&gt;x3 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>x2&#45;&gt;x3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.36,-72.41C45.09,-64.22 41.06,-54.14 37.38,-44.95\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"40.55,-43.45 33.59,-35.47 34.05,-46.05 40.55,-43.45\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.graphs.Digraph at 0x7f8ef0d06ed0>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Visualize the model\n",
    "scm.cgm.draw()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "balance on test set 0.5169333333333334, balance on test set 0.5152\n",
      "Epoch 0/9\n",
      "----------\n",
      "train Loss: 0.0725 Acc: 0.9716\n",
      "\n",
      "test Loss: 0.0608 Acc: 0.9776\n",
      "\n",
      "Epoch 1/9\n",
      "----------\n",
      "train Loss: 0.0371 Acc: 0.9853\n",
      "\n",
      "test Loss: 0.0255 Acc: 0.9888\n",
      "\n",
      "Epoch 2/9\n",
      "----------\n",
      "train Loss: 0.0342 Acc: 0.9872\n",
      "\n",
      "test Loss: 0.0268 Acc: 0.9868\n",
      "\n",
      "Epoch 3/9\n",
      "----------\n",
      "train Loss: 0.0322 Acc: 0.9869\n",
      "\n",
      "test Loss: 0.0276 Acc: 0.9884\n",
      "\n",
      "Epoch 4/9\n",
      "----------\n",
      "train Loss: 0.0299 Acc: 0.9888\n",
      "\n",
      "test Loss: 0.0201 Acc: 0.9912\n",
      "\n",
      "Epoch 5/9\n",
      "----------\n",
      "train Loss: 0.0285 Acc: 0.9891\n",
      "\n",
      "test Loss: 0.0188 Acc: 0.9920\n",
      "\n",
      "Epoch 6/9\n",
      "----------\n",
      "train Loss: 0.0271 Acc: 0.9897\n",
      "\n",
      "test Loss: 0.0162 Acc: 0.9928\n",
      "\n",
      "Epoch 7/9\n",
      "----------\n",
      "train Loss: 0.0279 Acc: 0.9897\n",
      "\n",
      "test Loss: 0.0192 Acc: 0.9928\n",
      "\n",
      "Epoch 8/9\n",
      "----------\n",
      "train Loss: 0.0267 Acc: 0.9895\n",
      "\n",
      "test Loss: 0.0474 Acc: 0.9816\n",
      "\n",
      "Epoch 9/9\n",
      "----------\n",
      "train Loss: 0.0241 Acc: 0.9899\n",
      "\n",
      "test Loss: 0.0591 Acc: 0.9756\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from carla.models.catalog import MLModelCatalog\n",
    "\n",
    "training_params = {\"lr\": 0.01, \"epochs\": 10, \"batch_size\": 16, \"hidden_size\": [18, 9, 3]}\n",
    "\n",
    "ml_model = MLModelCatalog(\n",
    "    dataset, model_type=\"ann\", load_online=False, backend=\"pytorch\"\n",
    ")\n",
    "ml_model.train(\n",
    "    learning_rate=training_params[\"lr\"],\n",
    "    epochs=training_params[\"epochs\"],\n",
    "    batch_size=training_params[\"batch_size\"],\n",
    "    hidden_size=training_params[\"hidden_size\"],\n",
    "    force_train=True\n",
    ")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   label        x1        x2        x3\n",
      "0    1.0 -3.822251  3.206103  1.394095\n",
      "1    0.0 -0.125186 -0.267489 -0.065593\n",
      "2    0.0  1.388819 -2.420645  0.119093\n",
      "3    0.0 -0.173873 -0.715379 -1.201855\n",
      "4    1.0 -0.637307  1.071684  0.379442\n",
      "5    0.0 -2.087104  2.218720 -0.248939\n",
      "6    0.0 -1.895168  2.386755 -0.718443\n",
      "7    0.0  0.721627 -0.507613 -0.951930\n",
      "8    0.0  1.772848 -1.068509 -0.722962\n",
      "9    1.0  1.783160 -0.904946  1.107616\n"
     ]
    }
   ],
   "source": [
    "from carla.models.negative_instances import predict_negative_instances\n",
    "# get factuals\n",
    "factuals = dataset.df#predict_negative_instances(ml_model, dataset.df)\n",
    "test_factual_with_labels = factuals.iloc[:10].reset_index(drop=True)\n",
    "test_factual=test_factual_with_labels.copy()\n",
    "print(test_factual)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''Causal Recourse Model '''\n",
    "from carla.recourse_methods.catalog.causal_recourse import (\n",
    "    CausalRecourse,\n",
    "    constraints,\n",
    "    samplers,\n",
    ")\n",
    "hyperparams = {\n",
    "    \"optimization_approach\": \"brute_force\",\n",
    "    \"num_samples\": 10,\n",
    "    \"scm\": scm,\n",
    "    \"constraint_handle\": constraints.point_constraint,\n",
    "    \"sampler_handle\": samplers.sample_true_m0,\n",
    "}\n",
    "\n",
    "#print(len(test_factual))\n",
    "# structural counterfactual (SCF)\n",
    "#cfs = CausalRecourse(ml_model, hyperparams).get_counterfactuals(test_factual)\n",
    "recourse=CausalRecourse(ml_model, hyperparams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "scm_output=carla.data.causal_model.CausalModel(\"sanity-3-lin-output\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.43.0 (0)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"122pt\" height=\"260pt\"\n",
       " viewBox=\"0.00 0.00 122.47 260.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 256)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-256 118.47,-256 118.47,4 -4,4\"/>\n",
       "<!-- x4 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>x4</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"41.47\" cy=\"-18\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"41.47\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\">x4</text>\n",
       "</g>\n",
       "<!-- x3 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>x3</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"41.47\" cy=\"-90\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"41.47\" y=\"-86.3\" font-family=\"Times,serif\" font-size=\"14.00\">x3</text>\n",
       "</g>\n",
       "<!-- x3&#45;&gt;x4 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>x3&#45;&gt;x4</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M41.47,-71.7C41.47,-63.98 41.47,-54.71 41.47,-46.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"44.97,-46.1 41.47,-36.1 37.97,-46.1 44.97,-46.1\"/>\n",
       "</g>\n",
       "<!-- x1 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>x1</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"32.47\" cy=\"-234\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"32.47\" y=\"-230.3\" font-family=\"Times,serif\" font-size=\"14.00\">x1</text>\n",
       "</g>\n",
       "<!-- x1&#45;&gt;x4 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>x1&#45;&gt;x4</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M24.86,-216.68C12.18,-187.23 -10.35,-123.47 5.47,-72 8.76,-61.31 15.01,-50.85 21.4,-42.08\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"24.36,-43.98 27.72,-33.93 18.83,-39.69 24.36,-43.98\"/>\n",
       "</g>\n",
       "<!-- x1&#45;&gt;x3 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>x1&#45;&gt;x3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M33.56,-215.87C35.09,-191.67 37.91,-147.21 39.74,-118.39\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"43.24,-118.39 40.38,-108.19 36.26,-117.95 43.24,-118.39\"/>\n",
       "</g>\n",
       "<!-- x2 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>x2</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"87.47\" cy=\"-162\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"87.47\" y=\"-158.3\" font-family=\"Times,serif\" font-size=\"14.00\">x2</text>\n",
       "</g>\n",
       "<!-- x1&#45;&gt;x2 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>x1&#45;&gt;x2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M44.41,-217.81C51.69,-208.55 61.14,-196.52 69.33,-186.09\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"72.14,-188.18 75.56,-178.16 66.63,-183.86 72.14,-188.18\"/>\n",
       "</g>\n",
       "<!-- x2&#45;&gt;x4 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>x2&#45;&gt;x4</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M87.95,-143.71C87.92,-125.35 86.27,-95.77 77.47,-72 73.6,-61.52 67.21,-51.09 60.89,-42.3\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"63.51,-39.97 54.69,-34.11 57.94,-44.2 63.51,-39.97\"/>\n",
       "</g>\n",
       "<!-- x2&#45;&gt;x3 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>x2&#45;&gt;x3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M77.04,-145.12C71.27,-136.34 63.99,-125.26 57.52,-115.42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"60.29,-113.26 51.87,-106.82 54.44,-117.1 60.29,-113.26\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.graphs.Digraph at 0x7f8ef09f6ed0>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Visualize the model\n",
    "scm_output.cgm.draw()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import importlib\n",
    "import Semantic_Meaningfulness.Semantic_Meaningfulness\n",
    "from Semantic_Meaningfulness.Semantic_Meaningfulness import Semantic\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>semantic</th>\n",
       "      <th>correct_relationships</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   semantic  correct_relationships\n",
       "0         1                    1.0\n",
       "1         1                    1.0\n",
       "2         1                    1.0\n",
       "3         0                    1.0\n",
       "4         1                    1.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "'''Original Evaluation'''\n",
    "from carla.data.catalog import CsvCatalog\n",
    "from carla import Benchmark\n",
    "mapping_dict={ \n",
    "      'u1': 'x1',\n",
    "      'u2': 'x2',\n",
    "    'u3': 'x3',\n",
    "      }\n",
    "# first initialize the benchmarking class by passing\n",
    "# black-box-model, recourse method, and factuals into it\n",
    "benchmark = Benchmark(ml_model, recourse, test_factual)\n",
    "\n",
    "# now you can decide if you want to run all measurements\n",
    "# or just specific ones.\n",
    "evaluation_measures = [\n",
    "    Semantic(ml_model, causal_graph_full=scm_output,causal_graph_small=scm),    \n",
    "]\n",
    "\n",
    "# now run all implemented measurements and create a\n",
    "# DataFrame which consists of all results\n",
    "results = benchmark.run_benchmark(evaluation_measures)\n",
    "\n",
    "display(results.head(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Causal Recourse - Semantic results 0.9 +/- 0.30000000000000004\n"
     ]
    }
   ],
   "source": [
    "# Averaging of the reults\n",
    "mean= np.mean(results['semantic'])\n",
    "std= np.std(results['semantic'])\n",
    "print(f'Causal Recourse - Semantic results {mean} +/- {std}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Causal Recourse - Semantic relationships 1.0 +/- 0.0\n"
     ]
    }
   ],
   "source": [
    "# Averaging of the reults\n",
    "mean= np.mean(results['correct_relationships'])\n",
    "std= np.std(results['correct_relationships'])\n",
    "print(f'Causal Recourse - Semantic relationships {mean} +/- {std}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.15 ('CARLA-7s4zdYsf')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.17"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "05951db0b7a8fb6113eda4ec48ea77d75ed33c5045908c39c7a5079655347ffc"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}