Fetch_data_for_Arabic_Dialect_Identification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Fetch data for Arabic Dialect Identification.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/youssefkamil/Arabic-Dialect-Identification/blob/main/Fetch_data_for_Arabic_Dialect_Identification.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import requests\n",
        "import json\n",
        "import pandas as pd"
      ],
      "metadata": {
        "id": "maDc5c5oH71T"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "url = 'https://recruitment.aimtechnologies.co/ai-tasks'\n",
        "myobj = [str(df['id'][0]), \"1057418989293485952\"]\n",
        "myobj=json.dumps(myobj)\n",
        "x = requests.post(url, data = myobj)\n",
        "x=x.json()"
      ],
      "metadata": {
        "id": "O1q0u_dkb5MZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x.items()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZkY6PfXEETO3",
        "outputId": "dc221894-8a0b-4722-9292-daaa6fedf148"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_items([('1175358310087892992', '@Nw8ieJUwaCAAreT لكن بالنهاية .. ينتفض .. يغير .'), ('1057418989293485952', '@mycousinvinnyys @hanyamikhail1 متهيالي دي شكولاته الهالوين  فين المحل ده')])"
            ]
          },
          "metadata": {},
          "execution_count": 154
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df=pd.read_csv('/content/dialect_dataset.csv')"
      ],
      "metadata": {
        "id": "wdcCn1R1HyIW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x=0\n",
        "data=[]\n",
        "ids=[]\n",
        "for i in range(len(df)):\n",
        "  if (i%1000==0) & (i >0) :\n",
        "\n",
        "    ids=json.dumps(ids)\n",
        "    x = requests.post(url, data = ids)\n",
        "    data.append(x.json())    \n",
        "    ids=[]\n",
        "    \n",
        "  ids.append(str(df['id'][i]))\n",
        "                        #################### to get last 198 ###########################\n",
        "ids=json.dumps(ids)                               \n",
        "x = requests.post(url, data = ids)\n",
        "data.append(x.json())"
      ],
      "metadata": {
        "id": "RQuguH6pKU1r"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['tweet']=''\n",
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "-6YOSxfkehSh",
        "outputId": "4fea9666-18a0-4bf9-e900-90598c9b2a51"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-3165ab34-c5d0-4ae1-ac45-45b86645071d\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>dialect</th>\n",
              "      <th>tweet</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1175358310087892992</td>\n",
              "      <td>IQ</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1175416117793349632</td>\n",
              "      <td>IQ</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1175450108898565888</td>\n",
              "      <td>IQ</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1175471073770573824</td>\n",
              "      <td>IQ</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1175496913145217024</td>\n",
              "      <td>IQ</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>458192</th>\n",
              "      <td>1019484980282580992</td>\n",
              "      <td>BH</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>458193</th>\n",
              "      <td>1021083283709407232</td>\n",
              "      <td>BH</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>458194</th>\n",
              "      <td>1017477537889431552</td>\n",
              "      <td>BH</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>458195</th>\n",
              "      <td>1022430374696239232</td>\n",
              "      <td>BH</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>458196</th>\n",
              "      <td>1022409931029458944</td>\n",
              "      <td>BH</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>458197 rows × 3 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3165ab34-c5d0-4ae1-ac45-45b86645071d')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-3165ab34-c5d0-4ae1-ac45-45b86645071d button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-3165ab34-c5d0-4ae1-ac45-45b86645071d');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                         id dialect tweet\n",
              "0       1175358310087892992      IQ      \n",
              "1       1175416117793349632      IQ      \n",
              "2       1175450108898565888      IQ      \n",
              "3       1175471073770573824      IQ      \n",
              "4       1175496913145217024      IQ      \n",
              "...                     ...     ...   ...\n",
              "458192  1019484980282580992      BH      \n",
              "458193  1021083283709407232      BH      \n",
              "458194  1017477537889431552      BH      \n",
              "458195  1022430374696239232      BH      \n",
              "458196  1022409931029458944      BH      \n",
              "\n",
              "[458197 rows x 3 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 175
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "FinalData=[]\n",
        "for ListJson in data:\n",
        "  for id, tweet in ListJson.items():\n",
        "    dialect=((df[df['id']==int(id)]['dialect']))\n",
        "    # print(dialect)\n",
        "    FinalData.append(  [ id, dialect.values[0]  , tweet ] )\n"
      ],
      "metadata": {
        "id": "kTPzGcqQdKBg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "FinalData[0:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XVYgxCgIo9HL",
        "outputId": "06e2d300-af55-44a2-8a37-ba6f7aa75043"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['1175358310087892992',\n",
              "  'IQ',\n",
              "  '@Nw8ieJUwaCAAreT لكن بالنهاية .. ينتفض .. يغير .'],\n",
              " ['1175416117793349632',\n",
              "  'IQ',\n",
              "  '@7zNqXP0yrODdRjK يعني هذا محسوب على البشر .. حيونه ووحشيه .. وتطلبون من الغرب يحترمكم ويؤمن بدينكم ولاينعتكم بالإرهاب ..'],\n",
              " ['1175450108898565888', 'IQ', '@KanaanRema مبين من كلامه خليجي'],\n",
              " ['1175471073770573824', 'IQ', '@HAIDER76128900 يسلملي مرورك وروحك الحلوه💐'],\n",
              " ['1175496913145217024', 'IQ', '@hmo2406 وين هل الغيبه  اخ محمد 🌸🌺'],\n",
              " ['1175668034146643968',\n",
              "  'IQ',\n",
              "  '@Badi9595 @KanaanRema يااخي الإرهابي اذا كان عراقي سعودي فلسطيني وين المشكلة على باب الفرض...خليجي وماعنده رحمه وين المساس بقدسيتك .. الله يرضى عنك خلصت مشاكل العرب واختلفنا بجنسيت ابو الطفل .. المغردين سألوا من وين ورجحت وين الكارثة اللي وصلتك'],\n",
              " ['1175670153884983296',\n",
              "  'IQ',\n",
              "  '@SarahNadhum90 @nUBNTdfVgACYQxV مطلبي يقدم استقالته وفوگاها اعتذار'],\n",
              " ['1175671762580856832',\n",
              "  'IQ',\n",
              "  '@KanaanRema @Badi9595 خلص والله لعيونكم انا ماعندي شيء معه بالعكس من متابعيني اللي بعتز فيهم ... خلص صافية لبن 🌸🙏🌷'],\n",
              " ['1175715664398561280',\n",
              "  'IQ',\n",
              "  '@SalahAlarbawi يمكن سؤال فات الكثير اللي يصور   شنو موقفه وكأنه يوثق بطوله .. المفروض حتى المصور يحال للقضاء لان نظرات الطفله تستنجد باتجاه الكاميرا..'],\n",
              " ['1176019816072777728',\n",
              "  'IQ',\n",
              "  '@Eng_alow91 @cb4LwpWrS1hT5lb @EdyCohen اولا اني ردت على رجل جنوبي واللي ذكر حجابها ... ثانياً انت شدعوه صايره محامي لكوهين ... صحيح مقولة العراق بلد الغرائب والعجائب']]"
            ]
          },
          "metadata": {},
          "execution_count": 245
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(FinalData)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qg4PyDzbOIcF",
        "outputId": "70adbcbb-ce0f-4fd5-957a-79e1d68f3d36"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "458197"
            ]
          },
          "metadata": {},
          "execution_count": 244
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.DataFrame(FinalData, columns =['id', 'dialect', 'tweet'])\n",
        "df.to_csv('/content/drive/MyDrive/Arabic Dialect/tweetsWithLabels.csv')"
      ],
      "metadata": {
        "id": "MEpF1qq2pBp2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.iloc[302336]['tweet']"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "MZCqnGyeq55S",
        "outputId": "071b14e7-c40b-45c0-c5bb-434f6a6940bf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'@haso1181 صحيح يا غالي وهو متوفر من فترة في محلات السوبرماركت الكبيرة المعروفة'"
            ]
          },
          "metadata": {},
          "execution_count": 259
        }
      ]
    }
  ]
}