Eurotec_CRM.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.5"
    },
    "colab": {
      "name": "Eurotec - CRM.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "ljtvhkWqhLp2",
        "elrPEUPuhLqE",
        "sg0hUC8EhLqO",
        "Nqxie5VCmr0G",
        "sIWS5Eq4spum",
        "o9DZi3Ebm_qA",
        "lmXro34LhbAP",
        "ItbNxY5Z2zhB",
        "Q_gt8dl4cf7n",
        "US47S0Xacnbq",
        "9_E01ym53Jz5",
        "Mtzj-yVmBhmR",
        "eQou87ogeY8s",
        "XROc7A2nepeE"
      ],
      "toc_visible": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "43fqSLv9iWHJ",
        "colab_type": "text"
      },
      "source": [
        "# **Google Drive credentials**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eigC0ebhiWAj",
        "colab_type": "code",
        "outputId": "13d7d7dd-ef9b-4982-b2b6-693eb5cbda40",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ljtvhkWqhLp2",
        "colab_type": "text"
      },
      "source": [
        "# **Loading libraries**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vuALQuGM6SA9",
        "colab_type": "code",
        "outputId": "5d28b3ef-8c24-47e1-fba1-772ff80764a0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!pip install unidecode"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: unidecode in /usr/local/lib/python3.6/dist-packages (1.1.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XjT3a7RvhLp6",
        "colab_type": "code",
        "outputId": "6719b872-fc5f-42fa-d98e-0d06bc7f2b90",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.model_selection import train_test_split\n",
        "from wordcloud import WordCloud, ImageColorGenerator\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from unicodedata import normalize\n",
        "from string import punctuation\n",
        "from ast import literal_eval\n",
        "from datetime import date\n",
        "from nltk import tokenize\n",
        "from nltk import ngrams\n",
        "from PIL import Image\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import unidecode\n",
        "import warnings\n",
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "nltk.download('rslp')\n",
        "\n",
        "warnings.filterwarnings(action='once')\n",
        "plt.rcParams[\"figure.figsize\"] = [16,9]\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package rslp to /root/nltk_data...\n",
            "[nltk_data]   Package rslp is already up-to-date!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "elrPEUPuhLqE",
        "colab_type": "text"
      },
      "source": [
        "# **Loading CSV**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xNdjYxTshLqH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "users_data = pd.read_csv(\"/content/drive/My Drive/Eurotec - CRM/users_data.csv\", \n",
        "                    sep=\";\").set_index('id')\n",
        "user_types_data = pd.read_csv(\"/content/drive/My Drive/Eurotec - CRM/user_types_data.csv\", \n",
        "                    sep=\";\").set_index('id')\n",
        "euroservices_data = pd.read_csv(\"/content/drive/My Drive/Eurotec - CRM/euroservices_data.csv\", \n",
        "                    sep=\";\").set_index('id')\n",
        "euroservice_types_data = pd.read_csv(\"/content/drive/My Drive/Eurotec - CRM/euroservice_types_data.csv\", \n",
        "                    sep=\";\").set_index('id')\n",
        "units_data = pd.read_csv(\"/content/drive/My Drive/Eurotec - CRM/units_data.csv\", \n",
        "                    sep=\";\").set_index('id')\n",
        "groups_data = pd.read_csv(\"/content/drive/My Drive/Eurotec - CRM/groups_data.csv\", \n",
        "                    sep=\";\").set_index('id')\n",
        "eurotec_mask = np.array(Image.open(\"/content/drive/My Drive/Eurotec - CRM/eurotec_logo.png\"))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sg0hUC8EhLqO",
        "colab_type": "text"
      },
      "source": [
        "# **Usuários por tipo de usuário**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eQBaHe-YhLqP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "users_data['user_type'] = user_types_data.loc[users_data['user_type_id']]['name'].values\n",
        "\n",
        "ax = sns.catplot(y='user_type', kind='count', data = users_data,orient=\"h\", height=6, aspect=11.7/6)\n",
        "ax.fig.suptitle('Relação tipo de usuário X Quantidade')\n",
        "ax.set_axis_labels(\"Quantidade\", \"Tipo de Usuário\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Nqxie5VCmr0G",
        "colab_type": "text"
      },
      "source": [
        "# **Euroservice por tipo de Euroservice**\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-AuOLsQChLqV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "euroservices_data['euroservice_type'] = euroservice_types_data.loc[euroservices_data['euroservice_type_id']]['name'].values\n",
        "\n",
        "# Filter euroservice data to after 2019 only\n",
        "euroservices_data = euroservices_data[euroservices_data['at_start'] > '2018-12-31']\n",
        "\n",
        "ax = sns.catplot(y='euroservice_type', kind='count', data = euroservices_data,orient=\"h\", height=6, aspect=11.7/6)\n",
        "ax.fig.suptitle('Relação tipo de Euroservice X Quantidade')\n",
        "ax.set_axis_labels(\"Quantidade\", \"Tipo de Euroservice\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sIWS5Eq4spum",
        "colab_type": "text"
      },
      "source": [
        "# **Euroservice por Grupo**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9GyDa293tDnh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        " # groups_data.loc[units_data.loc[72]['group_id']]\n",
        "euroservices_data['unit'] = units_data.loc[euroservices_data['unit_id']]['name'].values\n",
        "euroservices_data['group'] = groups_data.loc[units_data.loc[euroservices_data['unit_id']]['group_id'].values]['name'].values\n",
        "\n",
        "filter_mask = euroservices_data['group'] == 'Eurotec'\n",
        "\n",
        "ax = sns.catplot(y='group', kind='count', data = euroservices_data[~filter_mask], \n",
        "                 orient=\"h\", height=6, aspect=11.7/6,\n",
        "                 order = euroservices_data['group'].value_counts().index)\n",
        "ax.fig.suptitle('Relação de Euroservice por Grupo X Quantidade')\n",
        "ax.set_axis_labels(\"Quantidade\", \"Grupo\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JWUfXsJFwXDv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "euroservices_data_without_eurotec = euroservices_data[~filter_mask]\n",
        "ax = sns.countplot(y = \"group\", \n",
        "                   data = euroservices_data_without_eurotec,\n",
        "                   order = euroservices_data_without_eurotec['group'].value_counts().index)\n",
        "plt.suptitle('Relação de Euroservice por Grupo X Quantidade')\n",
        "plt.xlabel('Quantidade')\n",
        "plt.ylabel('Grupo')\n",
        "\n",
        "total = len(euroservices_data_without_eurotec['group'])\n",
        "for p in ax.patches:\n",
        "        percentage = '{:.0f} - {:.2f}%'.format(p.get_width(), 100 * p.get_width()/total)\n",
        "        x = p.get_x() + p.get_width() + 0.5\n",
        "        y = p.get_y() + p.get_height()/1.5\n",
        "        ax.annotate(percentage, (x, y))\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IjsaBRcXhWMD",
        "colab_type": "text"
      },
      "source": [
        "# **Euroservice por Usuário**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oc-Q6nIfmoSF",
        "colab_type": "text"
      },
      "source": [
        "#### **Transforma a coluna User_ids em array**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0RrYA_H5mn7B",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "euroservices_data['user_ids'] = euroservices_data['user_ids'].apply(literal_eval)\n",
        "euroservices_data['user_id'] = euroservices_data['user_id'].fillna(0).astype(int)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o9DZi3Ebm_qA",
        "colab_type": "text"
      },
      "source": [
        "#### **Dataframe usuários com Euroservices**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qY-EXYawnLKr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "users_euroservice_data_temp = pd.DataFrame(columns = ['users_ids', 'euroservice_id', 'user_id']) \n",
        "users_euroservice_data_temp['users_ids'] = euroservices_data['user_ids'].fillna(0)\n",
        "users_euroservice_data_temp['user_id'] = euroservices_data['user_id']\n",
        "users_euroservice_data_temp['euroservice_id'] = euroservices_data.index\n",
        "\n",
        "## Limpa a coluna e remove duplicatas\n",
        "for index, row in users_euroservice_data_temp.iterrows():\n",
        "    row['users_ids'].append(row['user_id'])\n",
        "\n",
        "users_euroservice_data_temp = users_euroservice_data_temp.drop(columns='user_id')\n",
        "for index, row in users_euroservice_data_temp.iterrows():\n",
        "    users_euroservice_data_temp.at[index,'users_ids'] = pd.unique(row['users_ids']).tolist()\n",
        "\n",
        "## Divide agora as linhas por user_id\n",
        "users_euroservice_data = users_euroservice_data_temp.explode('users_ids').reset_index().drop('id', axis=1).rename(columns={'users_ids': 'user_id'})\n",
        "users_euroservice_data = users_euroservice_data.drop(users_euroservice_data.query(\"user_id == 0\").index)\n",
        "users_euroservice_data['user_name'] = users_data.loc[users_euroservice_data['user_id']]['name'].values"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UZiyf7t6m4GS",
        "colab_type": "text"
      },
      "source": [
        "#### **Plotagem do gráfico**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p14nlY3aiou7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ax = sns.countplot(y = \"user_name\", \n",
        "                   data = users_euroservice_data,\n",
        "                   order = users_euroservice_data['user_name'].value_counts().index)\n",
        "\n",
        "plt.suptitle('Relação de Euroservice por Usuário X Quantidade')\n",
        "plt.xlabel('Quantidade')\n",
        "plt.ylabel('Usuário')\n",
        "total = len(users_euroservice_data['user_name'])\n",
        "for p in ax.patches:\n",
        "        percentage = '{:.0f} - {:.2f}%'.format(p.get_width(), 100 * p.get_width()/total)\n",
        "        x = p.get_x() + p.get_width() + 0.5\n",
        "        y = p.get_y() + p.get_height()/1.5\n",
        "        ax.annotate(percentage, (x, y))\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lmXro34LhbAP",
        "colab_type": "text"
      },
      "source": [
        "# **Euroservices com mais de um Usuário**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MWyJ12bwovIV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "euroservices_with_more_than_one_user = pd.concat(g for _, g in users_euroservice_data.groupby(\"euroservice_id\") if len(g) > 1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "96QqYfc1qkL7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "euroservices_with_duplications = pd.DataFrame([euroservices_data['unit_id'].count(), euroservices_with_more_than_one_user['euroservice_id'].value_counts().count()], columns = ['count'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T1gWL1gtl2mv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ax = euroservices_with_duplications.plot.bar(title=\"Euroservices - 2019\", legend=False, rot=0)\n",
        "y_pos = np.arange(len(euroservices_with_duplications['count']))\n",
        "plt.xticks(y_pos, ['Euroservice', 'Euroservice com mais de 1 técnico'])\n",
        "plt.ylabel('Quantidade')\n",
        "total = euroservices_with_duplications['count'].sum()\n",
        "for p in ax.patches:\n",
        "  percentage = '{:.0f} - {:.2f}%'.format(p.get_height(), 100 * p.get_height()/total)\n",
        "  ax.annotate(percentage, (p.get_x() + p.get_width() / 2., p.get_height()), ha='center', va='center', xytext=(0, 10), textcoords='offset points')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AbUUsg1Npv0o",
        "colab_type": "text"
      },
      "source": [
        "# **Trabalhando com os Dados de Euroservice**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2TKPJ9hwcFvT",
        "colab_type": "text"
      },
      "source": [
        "## Tempo médio para cada Euroservice"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZujM8l_5glfs",
        "colab_type": "text"
      },
      "source": [
        "#### Criação de coluna de tempo total de Euroservice"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FHi4pyT9gkj0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "euroservices_data['total_time'] = pd.to_datetime(euroservices_data['at_end']) - pd.to_datetime(euroservices_data['at_start'])\n",
        "euroservices_data['at_start'] = pd.to_datetime(euroservices_data['at_start'])\n",
        "euroservices_data['at_end'] = pd.to_datetime(euroservices_data['at_end'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "amMNxEHhcPcZ",
        "colab_type": "text"
      },
      "source": [
        "#### Plotagem do Gráfico"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ayvWPlMccNcB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tz_info = euroservices_data.iloc[0].at_start.tzinfo\n",
        "euroservices_data.sort_values(by=['at_start'], inplace=True)\n",
        "euroservices_datetime_data = euroservices_data[euroservices_data['total_time'] > '0 days']\n",
        "for month in range(1, 12):\n",
        "  df_plot = euroservices_datetime_data[euroservices_datetime_data['at_start'].map(lambda x: x.month) == month]\n",
        "  fig, ax = plt.subplots(figsize=(12,8))\n",
        "  ax.plot(df_plot['at_start'],\n",
        "          df_plot['total_time'])\n",
        "  ax.set(xlabel=\"Data\",   \n",
        "        ylabel=\"Dias\",\n",
        "        title=\"Tempo médio de Euroservice\")\n",
        "  plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "209BwVZHnuG6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "euroservices_datetime_data[]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J77sCYmsH_tg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "euroservices_datetime_data[euroservices_datetime_data['at_start'].map(lambda x: x.month) == 6]['total_time']\n",
        "euroservices_datetime_data.loc[2379]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b8qter8rDMAF",
        "colab_type": "text"
      },
      "source": [
        "### Quantidade de Euroservices com mais de 1 dia e menos de 1 dia"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oIaPGRBPDO2w",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "euroservices_data[euroservices_data['total_time'] > '1 days']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "anV-oreSqwcO",
        "colab_type": "text"
      },
      "source": [
        "## **Criar bag of words / word cloud na busca de recorrência de problemas**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ItbNxY5Z2zhB",
        "colab_type": "text"
      },
      "source": [
        "### **Texto em array**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ON0eT04vcbCj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "text = euroservices_data['description'].apply(lambda x: normalize('NFKD', x).encode('ASCII', 'ignore').decode('ASCII'))\n",
        "all_words = ' '.join([text for text in text])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q_gt8dl4cf7n",
        "colab_type": "text"
      },
      "source": [
        "### **Método de geração de Word Cloud**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l9ttUieFp4JZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def word_cloud(all_words):\n",
        "  word_cloud = WordCloud( background_color = \"white\",\n",
        "                          collocations = False,\n",
        "                          mode = \"RGB\",\n",
        "                          mask=eurotec_mask,\n",
        "                          normalize_plurals = True,\n",
        "                          random_state = 42).generate(all_words)\n",
        "  image_colors = ImageColorGenerator(eurotec_mask)\n",
        "  plt.figure(figsize=(20,20))\n",
        "  plt.imshow(word_cloud.recolor(color_func=image_colors), interpolation=\"bilinear\")\n",
        "  plt.axis('off')\n",
        "  plt.figure()\n",
        "  plt.show()\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "US47S0Xacnbq",
        "colab_type": "text"
      },
      "source": [
        "### **Método de Classificação do texto**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pWR-Wk6xcsES",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def text_classify(text):\n",
        "    vectorize = TfidfVectorizer(lowercase=False, ngram_range = (1,2))\n",
        "    bag_of_words = vectorize.fit_transform(text)\n",
        "    train, test, train_class, test_class = train_test_split(bag_of_words,\n",
        "                                                              text,\n",
        "                                                              random_state = 42)\n",
        "    logistic_regression = LogisticRegression(solver = \"lbfgs\")\n",
        "    logistic_regression.fit(train, train_class)\n",
        "    return logistic_regression.score(test, test_class)\n",
        "\n",
        "classification = pd.DataFrame({ text_classify(text) }, columns = [\"Classificação\"])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9_E01ym53Jz5",
        "colab_type": "text"
      },
      "source": [
        "### **Plotagem de Diagrama de Pareto**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g0zXb1F80eeJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def pareto(all_words, n):\n",
        "    space_token = tokenize.WhitespaceTokenizer()\n",
        "    token_text = space_token.tokenize(all_words)\n",
        "    frequency = nltk.FreqDist(token_text)\n",
        "    df_frequency = pd.DataFrame({\"Palavra\": list(frequency.keys()),\n",
        "                                   \"Frequência\": list(frequency.values())})\n",
        "    df_frequency = df_frequency.nlargest(columns = \"Frequência\", n = n)\n",
        "    plt.figure(figsize=(12,8))\n",
        "    ax = sns.barplot(data = df_frequency, x = \"Palavra\", y = \"Frequência\")\n",
        "    ax.set(ylabel = \"Contagem\")\n",
        "    plt.show()\n",
        "    \n",
        "pareto(all_words, 10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yVFIbMjN6dvI",
        "colab_type": "text"
      },
      "source": [
        "### **Tratamento dos dados (Palavras utilizadas nas descrições)**\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Mtzj-yVmBhmR",
        "colab_type": "text"
      },
      "source": [
        "#### **Remoção de palavras que não adicionam contexto, pontuação e acentuação**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SzgXJ0BI6fsN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "useless_words = nltk.corpus.stopwords.words(\"portuguese\")\n",
        "\n",
        "punctuation_token = tokenize.WordPunctTokenizer()\n",
        "\n",
        "punctuation_list = list()\n",
        "for p in punctuation:\n",
        "    punctuation_list.append(p)\n",
        "\n",
        "punctuation_stopwords = punctuation_list + useless_words\n",
        "punctuation_stopwords = [unidecode.unidecode(text) for text in punctuation_stopwords]\n",
        "\n",
        "processed_text = list()\n",
        "for word in text:\n",
        "    word = word.lower()\n",
        "    new_word = list()\n",
        "    text_word = punctuation_token.tokenize(word)\n",
        "    for another_word in text_word:\n",
        "        if another_word not in punctuation_stopwords:\n",
        "            new_word.append(another_word)\n",
        "    processed_text.append(' '.join(new_word))\n",
        "\n",
        "all_words_processed = ' '.join([processed_text for processed_text in processed_text]) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eQou87ogeY8s",
        "colab_type": "text"
      },
      "source": [
        "### **Plotagem dos dados tratados**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ygQc6qcwB5Nj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pareto(all_words_processed, 10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2ueW20j1ehv1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "word_cloud(all_words_processed)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XROc7A2nepeE",
        "colab_type": "text"
      },
      "source": [
        "### **Histórico de confiança dos dados**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wo26MbU3CKwi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "classification = classification.append(pd.DataFrame({ text_classify(processed_text) }, columns = [\"Classificação\"]))\n",
        "sns.lineplot(x=\"index\",  y=\"Classificação\", data=classification.reset_index(drop=True).reset_index())\n",
        "plt.suptitle(\"Timeline de classificações de Word Cloud\")"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}