natural images small resnet.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Steps\n",
    "[ ] Run inferences on a different dataset and use activations to calculate one-hot concept vector for each image  \n",
    "[ ] Store prediction as label  \n",
    "[ ] Train decision tree  \n",
    "[ ] Get metrics for accuracy of decision tree predictions and actual alexnet predictions  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "from __future__ import division\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import numpy as np\n",
    "import torchvision\n",
    "from torch import randperm\n",
    "from torch._utils import _accumulate\n",
    "from torchvision import datasets, models, transforms\n",
    "from torch.utils.data import DataLoader, sampler\n",
    "from torch.utils.data.dataset import Subset\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import os\n",
    "import copy\n",
    "from tqdm import tqdm, tqdm_notebook\n",
    "import pandas as pd\n",
    "\n",
    "import graphviz\n",
    "\n",
    "\n",
    "\n",
    "import skimage\n",
    "\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.preprocessing import normalize\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn import tree\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_split(dataset, lengths):\n",
    "    \"\"\"\n",
    "    Randomly split a dataset into non-overlapping new datasets of given lengths.\n",
    "\n",
    "    Arguments:\n",
    "        dataset (Dataset): Dataset to be split\n",
    "        lengths (sequence): lengths of splits to be produced\n",
    "    \"\"\"\n",
    "    if sum(lengths) != len(dataset):\n",
    "        raise ValueError(\"Sum of input lengths does not equal the length of the input dataset!\")\n",
    "\n",
    "    indices = randperm(sum(lengths))\n",
    "    return [Subset(dataset, indices[offset - length:offset]) for offset, length in zip(_accumulate(lengths), lengths)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = 'data/natural_images_small'\n",
    "\n",
    "model_name = 'resnet50'\n",
    "\n",
    "num_classes = 4\n",
    "\n",
    "batch_size = 10\n",
    "\n",
    "num_epochs = 5\n",
    "\n",
    "feature_extract = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, dataloaders, criterion, optimizer, num_epochs=25, is_inception=False):\n",
    "    since = time.time()\n",
    "\n",
    "    val_acc_history = []\n",
    "\n",
    "    best_model_wts = copy.deepcopy(model.state_dict())\n",
    "    best_acc = 0.0\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n",
    "        print('-' * 10)\n",
    "\n",
    "        # Each epoch has a training and validation phase\n",
    "        for phase in ['train', 'val']:\n",
    "            if phase == 'train':\n",
    "                model.train()  # Set model to training mode\n",
    "            else:\n",
    "                model.eval()   # Set model to evaluate mode\n",
    "\n",
    "            running_loss = 0.0\n",
    "            running_corrects = 0\n",
    "\n",
    "            # Iterate over data.\n",
    "            for inputs, labels in dataloaders[phase]:\n",
    "                inputs = inputs.to(device)\n",
    "                labels = labels.to(device)\n",
    "\n",
    "                # zero the parameter gradients\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                # forward\n",
    "                # track history if only in train\n",
    "                with torch.set_grad_enabled(phase == 'train'):\n",
    "                    # Get model outputs and calculate loss\n",
    "                    # Special case for inception because in training it has an auxiliary output. In train\n",
    "                    #   mode we calculate the loss by summing the final output and the auxiliary output\n",
    "                    #   but in testing we only consider the final output.\n",
    "                    if is_inception and phase == 'train':\n",
    "                        # From https://discuss.pytorch.org/t/how-to-optimize-inception-model-with-auxiliary-classifiers/7958\n",
    "                        outputs, aux_outputs = model(inputs)\n",
    "                        loss1 = criterion(outputs, labels)\n",
    "                        loss2 = criterion(aux_outputs, labels)\n",
    "                        loss = loss1 + 0.4*loss2\n",
    "                    else:\n",
    "                        outputs = model(inputs)\n",
    "                        loss = criterion(outputs, labels)\n",
    "\n",
    "                    _, preds = torch.max(outputs, 1)\n",
    "\n",
    "                    # backward + optimize only if in training phase\n",
    "                    if phase == 'train':\n",
    "                        loss.backward()\n",
    "                        optimizer.step()\n",
    "\n",
    "                # statistics\n",
    "                running_loss += loss.item() * inputs.size(0)\n",
    "                running_corrects += torch.sum(preds == labels.data)\n",
    "\n",
    "            epoch_loss = running_loss / len(dataloaders[phase].dataset)\n",
    "            epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)\n",
    "\n",
    "            print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n",
    "\n",
    "            # deep copy the model\n",
    "            if phase == 'val' and epoch_acc > best_acc:\n",
    "                best_acc = epoch_acc\n",
    "                best_model_wts = copy.deepcopy(model.state_dict())\n",
    "            if phase == 'val':\n",
    "                val_acc_history.append(epoch_acc)\n",
    "\n",
    "        print()\n",
    "\n",
    "    time_elapsed = time.time() - since\n",
    "    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n",
    "    print('Best val Acc: {:4f}'.format(best_acc))\n",
    "\n",
    "    # load best model weights\n",
    "    model.load_state_dict(best_model_wts)\n",
    "    return model, val_acc_history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_parameter_requires_grad(model, feature_extracting):\n",
    "    if feature_extracting:\n",
    "        for param in model.parameters():\n",
    "            param.requires_grad = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize the model\n",
    "model = models.resnet50(pretrained=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_parameter_requires_grad(model, feature_extract)\n",
    "num_ftrs = model.fc.in_features\n",
    "\n",
    "# change classifier layer\n",
    "model.fc = nn.Linear(num_ftrs, num_classes)\n",
    "input_size = 224 # how large the image should be"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing Datasets and Dataloaders...\n"
     ]
    }
   ],
   "source": [
    "# Data augmentation and normalization for training\n",
    "# Just normalization for validation\n",
    "data_transforms = {\n",
    "    'train': transforms.Compose([\n",
    "        transforms.RandomResizedCrop(input_size),\n",
    "        transforms.RandomHorizontalFlip(),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "    'val': transforms.Compose([\n",
    "        transforms.Resize(input_size),\n",
    "        transforms.CenterCrop(input_size),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "}\n",
    "\n",
    "print(\"Initializing Datasets and Dataloaders...\")\n",
    "\n",
    "# # Create training and validation datasets\n",
    "# image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x), data_transforms[x]) for x in ['train', 'val']}\n",
    "# # Create training and validation dataloaders\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2799 700\n"
     ]
    }
   ],
   "source": [
    "all_data = datasets.ImageFolder(root=data_dir)\n",
    "train_data_len = int(len(all_data)*0.8)\n",
    "valid_data_len = int((len(all_data) - train_data_len))\n",
    "train_data, val_data = random_split(all_data, [train_data_len, valid_data_len])\n",
    "train_data.dataset.transform = data_transforms['train']\n",
    "val_data.dataset.transform = data_transforms['val']\n",
    "print(len(train_data), len(val_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataloaders = {\n",
    "    'train': torch.utils.data.DataLoader(train_data, batch_size=batch_size, shuffle=False, num_workers=4), \n",
    "    'val': torch.utils.data.DataLoader(val_data, batch_size=batch_size, shuffle=False, num_workers=4) \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model initialized and moved to GPU\n"
     ]
    }
   ],
   "source": [
    "device = torch.device('cuda:0')\n",
    "model = model.to(device)\n",
    "print(\"model initialized and moved to GPU\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Params to learn:\n",
      "\t fc.weight\n",
      "\t fc.bias\n"
     ]
    }
   ],
   "source": [
    "# Gather the parameters to be optimized/updated in this run. If we are\n",
    "#  finetuning we will be updating all parameters. However, if we are\n",
    "#  doing feature extract method, we will only update the parameters\n",
    "#  that we have just initialized, i.e. the parameters with requires_grad\n",
    "#  is True.\n",
    "params_to_update = model.parameters()\n",
    "print(\"Params to learn:\")\n",
    "params_to_update = []\n",
    "for name,param in model.named_parameters():\n",
    "    if param.requires_grad == True:\n",
    "        params_to_update.append(param)\n",
    "        print(\"\\t\",name)\n",
    "        \n",
    "optimizer_ft = optim.SGD(params_to_update, lr=0.001, momentum=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0/4\n",
      "----------\n",
      "train Loss: 0.1742 Acc: 0.9589\n",
      "val Loss: 0.0177 Acc: 0.9986\n",
      "\n",
      "Epoch 1/4\n",
      "----------\n",
      "train Loss: 0.0570 Acc: 0.9879\n",
      "val Loss: 0.0100 Acc: 0.9986\n",
      "\n",
      "Epoch 2/4\n",
      "----------\n",
      "train Loss: 0.0423 Acc: 0.9921\n",
      "val Loss: 0.0074 Acc: 0.9986\n",
      "\n",
      "Epoch 3/4\n",
      "----------\n",
      "train Loss: 0.0345 Acc: 0.9925\n",
      "val Loss: 0.0063 Acc: 0.9986\n",
      "\n",
      "Epoch 4/4\n",
      "----------\n",
      "train Loss: 0.0294 Acc: 0.9939\n",
      "val Loss: 0.0056 Acc: 0.9986\n",
      "\n",
      "Training complete in 0m 55s\n",
      "Best val Acc: 0.998571\n"
     ]
    }
   ],
   "source": [
    "# Setup the loss fxn\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "dataloaders_dict = {}\n",
    "# Train and evaluate\n",
    "model, hist = train_model(\n",
    "        model, dataloaders, criterion, optimizer_ft, \n",
    "        num_epochs=num_epochs, is_inception=(model_name==\"inception\")\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save trained model\n",
    "torch.save(model.state_dict(), 'models/resnet50-natural-images-small.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# max pool, flattens, then normalizes activations\n",
    "def transform_activations(activations):\n",
    "    m = nn.AvgPool2d(4)\n",
    "    max_pool_outputs = m(activations)\n",
    "    flattened_outputs = max_pool_outputs.reshape((max_pool_outputs.shape[0], np.prod(max_pool_outputs.shape[1:])))\n",
    "    normalized_outputs = normalize(flattened_outputs)\n",
    "    return normalized_outputs\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "activations = []\n",
    "classes = []\n",
    "# define layer hook\n",
    "def layer_hook(module, input, output):\n",
    "    layer_output = output.data.cpu()\n",
    "    activations.append(layer_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "layer = 'layer3'\n",
    "# register hook on layer\n",
    "# handle = model.layer1[2].bn3.register_forward_hook(layer_hook)\n",
    "# handle = model.layer2[3].bn3.register_forward_hook(layer_hook)\n",
    "handle = model.layer3[5].bn3.register_forward_hook(layer_hook)\n",
    "# handle = model.layer4[2].bn3.register_forward_hook(layer_hook)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "activations = []\n",
    "classes = []\n",
    "\n",
    "activation_dict = {}\n",
    "\n",
    "# setup for evaluation mode\n",
    "model.eval()\n",
    "\n",
    "# do same thing for training set\n",
    "for inputs, labels in dataloaders['train']:\n",
    "    inputs = inputs.to(device)\n",
    "    outputs = model(inputs).data.cpu()\n",
    "    _, preds = torch.max(outputs, 1)\n",
    "\n",
    "    classes.append(preds)\n",
    "\n",
    "activations = torch.cat(activations, dim=0)\n",
    "classes = torch.cat(classes, dim=0)\n",
    "\n",
    "activation_dict['train'] = (transform_activations(activations), classes)\n",
    "\n",
    "\n",
    "# reset and do the same for val set\n",
    "activations = []\n",
    "classes = []\n",
    "\n",
    "# do same thing for val set\n",
    "for inputs, labels in dataloaders['val']:\n",
    "    inputs = inputs.to(device)\n",
    "    outputs = model(inputs).data.cpu()\n",
    "    _, preds = torch.max(outputs, 1)\n",
    "\n",
    "    classes.append(preds)\n",
    "\n",
    "activations = torch.cat(activations, dim=0)\n",
    "classes = torch.cat(classes, dim=0)\n",
    "\n",
    "activation_dict['val'] = (transform_activations(activations), classes)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "handle.remove()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "activations = activation_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model in evaluation mode...\n"
     ]
    }
   ],
   "source": [
    "model.eval()\n",
    "print(\"model in evaluation mode...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2799, 9216)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "activations['train'][0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('pca/%s.pickle' % layer, 'rb') as handle:\n",
    "    pca = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_acts = pca.transform(activations['train'][0])\n",
    "val_acts = pca.transform(activations['val'][0])\n",
    "Y_train = activations['train'][1]\n",
    "Y_val = activations['val'][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "# Load data (deserialize)\n",
    "with open('classifiers-%s.pickle' % layer, 'rb') as handle:\n",
    "    classifiers = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "concepts_to_remove = [93, 19, 38, 79]\n",
    "for c in concepts_to_remove:\n",
    "    if classifiers.get(c) is not None:\n",
    "        del classifiers[c]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "concepts = list(classifiers.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "97"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(concepts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_concept_vectors(activations, classifiers, concepts=[]):\n",
    "    predictions = []\n",
    "    \n",
    "    # if you specify which concepts to measure:\n",
    "    if len(concepts) > 0:\n",
    "        for concept in concepts:\n",
    "            clf = classifiers.get(concept)[0]\n",
    "            pred = clf.predict(activations)[np.newaxis]\n",
    "            pred = np.transpose(pred)\n",
    "            predictions.append(pred)\n",
    "\n",
    "    else:\n",
    "        num_concepts = len(classifiers.keys())\n",
    "\n",
    "        # for each classifier, get the prediction\n",
    "        for clf in classifiers.items():\n",
    "            pred = clf[1][0].predict(activations)[np.newaxis]\n",
    "            pred = np.transpose(pred)\n",
    "            predictions.append(pred)\n",
    "    \n",
    "    return np.hstack(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter_concepts = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,44,48,52,57,60]\n",
    "# filter_concepts = list(set(filter_concepts) & set(concepts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = get_concept_vectors(train_acts, classifiers)\n",
    "X_val = get_concept_vectors(val_acts, classifiers)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_frame = pd.read_csv('dataset/broden1_227/label.csv')\n",
    "concept_names = [labels_frame.at[x-1, 'name'] for x in concepts]\n",
    "class_names = train_data.dataset.classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['car', 'dog', 'flower', 'person']"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "97"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(concepts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for n in range(20):#len(X_train):\n",
    "#     names = [concept_names[i] for i, x in enumerate(X_train[n]) if x > 0]\n",
    "#     print(len(names),names,class_names[Y_train[n]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_tree(name, X_train, X_val, Y_train, Y_val, min_samples, max_depth, concept_names, class_names, export=False, sample_weights=None):\n",
    "    decision_tree = tree.DecisionTreeClassifier(min_samples_leaf=min_samples, max_depth=max_depth, criterion='gini', min_impurity_split=0.1)\n",
    "    decision_tree = decision_tree.fit(X_train, Y_train, sample_weight=sample_weights)\n",
    "    score = decision_tree.score(X_val, Y_val)\n",
    "    print(\"%f\" % score)\n",
    "    dot_data = tree.export_graphviz(\n",
    "        decision_tree, out_file=None, feature_names=concept_names, class_names=train_data.dataset.classes, \n",
    "        filled=True, leaves_parallel=False, impurity=False, rounded=True, label='none'\n",
    "    )\n",
    "    graph = graphviz.Source(dot_data)\n",
    "    \n",
    "    important_features = decision_tree.feature_importances_\n",
    "    \n",
    "    if export:\n",
    "        file_name = \"%s-%i-%i-%s\" % (name, max_depth, min_samples, (\"%.4f\" % round(score,4)).split('.')[1])\n",
    "        print(\"outputting file to %s\" % file_name)\n",
    "        graph.render(\"trees/natural_images_small/%s/%s\"% (layer, file_name))\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_sample_weights(data):\n",
    "    weights = []\n",
    "    scores = [value[1] for key, value in classifiers.items()]\n",
    "    for d in data:\n",
    "        avg_sum = 0\n",
    "        count = 0\n",
    "        avg = 0\n",
    "        for i, concept in enumerate(d):\n",
    "\n",
    "            if (concept == 1):\n",
    "                avg_sum += scores[i]\n",
    "                count+=1\n",
    "        if(count != 0):            \n",
    "            avg = avg_sum / count\n",
    "            \n",
    "        if(count == 0):\n",
    "            weights.append(0)\n",
    "        else:\n",
    "            weights.append(avg)\n",
    "    \n",
    "    return weights\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_weights=generate_sample_weights(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.912857\n",
      "outputting file to natural_images_small-5-20-9129\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/conner/anaconda3/envs/torch2/lib/python2.7/site-packages/sklearn/tree/tree.py:283: DeprecationWarning: The min_impurity_split parameter is deprecated. Its default value will change from 1e-7 to 0 in version 0.23, and it will be removed in 0.25. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "MIN_SAMPLES = 20\n",
    "MAX_DEPTH = 5\n",
    "EXPORT = True\n",
    "# train_tree('natural_images_small_layer3_weighted', X_train, X_val, Y_train, Y_val, MIN_SAMPLES, MAX_DEPTH, concept_names, class_names, export=EXPORT, sample_weights=sample_weights)\n",
    "train_tree('natural_images_small', X_train, X_val, Y_train, Y_val, MIN_SAMPLES, MAX_DEPTH, concept_names, class_names, export=EXPORT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/conner/anaconda3/envs/torch2/lib/python2.7/site-packages/sklearn/tree/tree.py:283: DeprecationWarning: The min_impurity_split parameter is deprecated. Its default value will change from 1e-7 to 0 in version 0.23, and it will be removed in 0.25. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "/home/conner/anaconda3/envs/torch2/lib/python2.7/site-packages/sklearn/tree/tree.py:283: DeprecationWarning: The min_impurity_split parameter is deprecated. Its default value will change from 1e-7 to 0 in version 0.23, and it will be removed in 0.25. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "/home/conner/anaconda3/envs/torch2/lib/python2.7/site-packages/sklearn/tree/tree.py:283: DeprecationWarning: The min_impurity_split parameter is deprecated. Its default value will change from 1e-7 to 0 in version 0.23, and it will be removed in 0.25. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "/home/conner/anaconda3/envs/torch2/lib/python2.7/site-packages/sklearn/tree/tree.py:283: DeprecationWarning: The min_impurity_split parameter is deprecated. Its default value will change from 1e-7 to 0 in version 0.23, and it will be removed in 0.25. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "/home/conner/anaconda3/envs/torch2/lib/python2.7/site-packages/sklearn/tree/tree.py:283: DeprecationWarning: The min_impurity_split parameter is deprecated. Its default value will change from 1e-7 to 0 in version 0.23, and it will be removed in 0.25. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "/home/conner/anaconda3/envs/torch2/lib/python2.7/site-packages/sklearn/tree/tree.py:283: DeprecationWarning: The min_impurity_split parameter is deprecated. Its default value will change from 1e-7 to 0 in version 0.23, and it will be removed in 0.25. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "/home/conner/anaconda3/envs/torch2/lib/python2.7/site-packages/sklearn/tree/tree.py:283: DeprecationWarning: The min_impurity_split parameter is deprecated. Its default value will change from 1e-7 to 0 in version 0.23, and it will be removed in 0.25. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "/home/conner/anaconda3/envs/torch2/lib/python2.7/site-packages/sklearn/tree/tree.py:283: DeprecationWarning: The min_impurity_split parameter is deprecated. Its default value will change from 1e-7 to 0 in version 0.23, and it will be removed in 0.25. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "/home/conner/anaconda3/envs/torch2/lib/python2.7/site-packages/sklearn/tree/tree.py:283: DeprecationWarning: The min_impurity_split parameter is deprecated. Its default value will change from 1e-7 to 0 in version 0.23, and it will be removed in 0.25. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "/home/conner/anaconda3/envs/torch2/lib/python2.7/site-packages/sklearn/tree/tree.py:283: DeprecationWarning: The min_impurity_split parameter is deprecated. Its default value will change from 1e-7 to 0 in version 0.23, and it will be removed in 0.25. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.894286\n",
      "0.908571\n",
      "0.918571\n",
      "0.924286\n",
      "0.927143\n",
      "0.927143\n",
      "0.894286\n",
      "0.908571\n",
      "0.900000\n",
      "0.900000\n",
      "0.900000\n",
      "0.900000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/conner/anaconda3/envs/torch2/lib/python2.7/site-packages/sklearn/tree/tree.py:283: DeprecationWarning: The min_impurity_split parameter is deprecated. Its default value will change from 1e-7 to 0 in version 0.23, and it will be removed in 0.25. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "/home/conner/anaconda3/envs/torch2/lib/python2.7/site-packages/sklearn/tree/tree.py:283: DeprecationWarning: The min_impurity_split parameter is deprecated. Its default value will change from 1e-7 to 0 in version 0.23, and it will be removed in 0.25. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "min_sample_vals = [5, 20]\n",
    "max_depth_vals = [3, 4, 5, 10, 20, 50]\n",
    "\n",
    "for min_sample in min_sample_vals:\n",
    "    for max_depth in max_depth_vals:\n",
    "        train_tree('natural_images_small', X_train, X_val, Y_train, Y_val, min_sample, max_depth, concept_names, class_names, export=EXPORT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "97"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(concepts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from matplotlib import colors\n",
    "from matplotlib.ticker import PercentFormatter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy_df = pd.DataFrame(data=([concept_names[c] for c in concepts[:25]], [value[1] for key, value in classifiers.items()][:25]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.bar([concept_names[c] for c in concepts[:20]], [value[1] for key, value in classifiers.items()][:20], width=0.5, align='center')\n",
    "plt.title('Acurracies of Top 20 Concepts for Layer 3 of Resnet50')\n",
    "plt.xlabel('Concepts')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xticks(rotation=80)\n",
    "plt.savefig('test.pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(SGDClassifier(alpha=0.0001, average=True, class_weight=None,\n",
       "        early_stopping=True, epsilon=0.1, eta0=1.5, fit_intercept=True,\n",
       "        l1_ratio=0.15, learning_rate='optimal', loss='hinge',\n",
       "        max_iter=150.0, n_iter=None, n_iter_no_change=5, n_jobs=None,\n",
       "        penalty='l2', power_t=0.5, random_state=None, shuffle=True,\n",
       "        tol=None, validation_fraction=0.1, verbose=0, warm_start=False),\n",
       " 0.8942307692307693)"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifiers.get(29)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (torch2)",
   "language": "python",
   "name": "myenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}