Scrapper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f4155a03",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python\n",
    "# coding: utf-8\n",
    "\n",
    "from selenium import webdriver\n",
    "from bs4 import BeautifulSoup\n",
    "from selenium.webdriver.chrome.service import Service as ChromeService\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "from selenium.webdriver.common.by import By\n",
    "from langdetect import detect_langs\n",
    "import pandas as pd\n",
    "import time\n",
    "import random\n",
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "68c5f4b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import import_ipynb\n",
    "import config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d32ae51a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "importing Jupyter notebook from Model.ipynb\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "import import_ipynb\n",
    "import Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc20ba74",
   "metadata": {},
   "outputs": [],
   "source": [
    "number_of_chats=0\n",
    "pipelines_dict = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56bbc954",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# We open Chrome browser in incognito mode and maximized\n",
    "def open_browser():\n",
    "    service = ChromeService(executable_path=ChromeDriverManager().install())\n",
    "\n",
    "    chrome_options = webdriver.ChromeOptions()\n",
    "    chrome_options.add_argument(\"--incognito\")\n",
    "    chrome_options.add_argument(\"--start-maximized\")\n",
    "\n",
    "    driver = webdriver.Chrome(service=service, chrome_options=chrome_options)\n",
    "    return driver\n",
    "\n",
    "\n",
    "# Link to the desired page to be scraped\n",
    "def navigate(driver):\n",
    "    driver.get('https://www.linkedin.com/login/es?fromSignIn=true&trk=guest_homepage-basic_nav-header-signin');\n",
    "    time.sleep(10)\n",
    "\n",
    "    # Save all the content in the \"soup\" variable\n",
    "    content = driver.page_source\n",
    "    soup = BeautifulSoup(content, \"html.parser\")\n",
    "\n",
    "\n",
    "#get and insert login credentials\n",
    "def login(driver):\n",
    "  \n",
    "    # Login\n",
    "    # Getting the username element \n",
    "    username_box = driver.find_element_by_id(\"username\")  \n",
    "    username_box.clear()\n",
    "    time.sleep(1)\n",
    "\n",
    "    # Sending the keys for username     \n",
    "    username_box.send_keys(config.username) \n",
    "    time.sleep(1)\n",
    "\n",
    "\n",
    "    # Getting the password element                            \n",
    "    password_box = driver.find_element_by_id(\"password\") \n",
    "    password_box.clear()\n",
    "    time.sleep(1)\n",
    "\n",
    "    # Sending the keys for password   \n",
    "    password_box.send_keys(config.password) \n",
    "    time.sleep(1)\n",
    "\n",
    "    # Getting the tag for submit button           \n",
    "    driver.find_element(By.XPATH, \"//button[@aria-label='Iniciar sesión']\").click()\n",
    "    time.sleep(10)\n",
    "\n",
    "\n",
    "#Search IT recruiters\n",
    "def search(driver):\n",
    "    search_box=driver.find_element(By.XPATH, \"//input[@placeholder='Buscar']\")\n",
    "    try:\n",
    "        search_box.clear()\n",
    "    except:\n",
    "        print(\"No content to clear\")\n",
    "\n",
    "    search_box.send_keys(\"recruiter meta\")\n",
    "    time.sleep(5)\n",
    "    \n",
    "    #Click see all contacts\n",
    "    driver.find_element(By.XPATH, \"//span[@class='search-global-typeahead__hit-text t-16 t-black']\").click()\n",
    "    time.sleep(5)\n",
    "    \n",
    "    #set filter to person\n",
    "    try:\n",
    "        driver.find_element(By.XPATH, \"//button[@aria-label='Personas']\").click()\n",
    "        time.sleep(1)\n",
    "    except:\n",
    "        driver.find_element(By.XPATH, \"//*[@id='search-reusables__filters-bar']/ul/li[1]/button\").click()\n",
    "        time.sleep(1)\n",
    "\n",
    "\n",
    "# Save all the content in the \"soup\" variable\n",
    "def get_page_content(driver):\n",
    "    content = driver.page_source\n",
    "    soup = BeautifulSoup(content, \"html.parser\")\n",
    "\n",
    "\n",
    "def send_invitations(num, driver):\n",
    "    \n",
    "    i=0\n",
    "    contacted=0\n",
    "    while contacted<int(num):\n",
    "        content = driver.page_source\n",
    "        soup = BeautifulSoup(content)\n",
    "        new = soup.findAll('div', {'class':'entity-result'})\n",
    "\n",
    "        # Iterates over all the persons in this page (10)\n",
    "        for item in new:\n",
    "            print(\"######################\" + \"Number: \"+ str(i) + \" Person\"  +   \"###########################\")\n",
    "            \n",
    "            #location=driver.find_element(By.XPATH,\"entity-result__secondary-subtitle t-14 t-normal\")\n",
    "            soup=get_content(driver)\n",
    "            print(\"RECRUITER FROM\")\n",
    "            print(soup.find('div',{'class':'entity-result__secondary-subtitle t-14 t-normal'}).get_text())\n",
    "\n",
    "            #Press connect button, random time (to imitate human behaviour)\n",
    "            time.sleep(random.randint(2, 7))\n",
    "            connect_button=None\n",
    "            follow_button=None\n",
    "            message_button=None\n",
    "            try:\n",
    "                #connect_button=driver.find_element(By.XPATH,\"//button[contains(@class, 'artdeco-button artdeco-button--2 artdeco-button--secondary ember-view') and contains(., 'Conectar')]\")\n",
    "                connect_button=driver.find_element(By.XPATH,\"//button[contains(@class, 'artdeco-button artdeco-button--2 artdeco-button--secondary ember-view') and contains(., 'Conectar')]\")\n",
    "                #connect_button.click()\n",
    "                connect=1\n",
    "                contacted=contacted+1\n",
    "                print(\"Connect button detected\")\n",
    "            except:\n",
    "                try:\n",
    "                    follow_button=driver.find_element(By.XPATH,\"//button[contains(@class, 'artdeco-button artdeco-button--2 artdeco-button--secondary ember-view') and contains(., 'Seguir')]\")\n",
    "                    follow_button.click()\n",
    "                    print(\"Follow button detected\")\n",
    "                except:\n",
    "                    try:\n",
    "                        message_button=driver.find_element(By.XPATH,\"//button[contains(@class, 'artdeco-button artdeco-button--2 artdeco-button--secondary ember-view') and contains(., 'Mensaje')]\")\n",
    "                        #message_button.click()\n",
    "                        print(\"Message button detected\")\n",
    "                    except:\n",
    "                        if connect_button==None and follow_button==None and message_button==None:\n",
    "                            print(\"[+] No button detected\\n\")\n",
    "\n",
    "            #print(conectar_button)\n",
    "            time.sleep(random.randint(2, 7))\n",
    "\n",
    "            #Press send invitation\n",
    "            try:\n",
    "                enviar_button=driver.find_element(By.XPATH,\"//button[contains(@class, 'artdeco-button artdeco-button--2 artdeco-button--primary ember-view ml1') and contains(., 'Enviar')]\")\n",
    "                #enviar_button.click()\n",
    "                print(\"Invitation sent\")\n",
    "                \n",
    "                #To be sure that the button pressed before this was \"connect\"\n",
    "                if connect==1:\n",
    "                    contacted=contacted+1\n",
    "            except:\n",
    "                print(\"[+] No 'send' button \\n\")\n",
    "\n",
    "                \n",
    "            time.sleep(2)\n",
    "            #print(enviar_button)\n",
    "\n",
    "            if contacted<int(num):\n",
    "                # Keep going\n",
    "                i=i+1\n",
    "            else:\n",
    "                #Number of recruiters contacted reached\n",
    "                break\n",
    "                \n",
    "        #Scroll down to click next\n",
    "        driver.execute_script(\"window.scrollTo(0, document.body.scrollHeight);\")\n",
    "        time.sleep(2)\n",
    "        \n",
    "        # Save all the content in the \"soup\" variable\n",
    "        content = driver.page_source\n",
    "        soup = BeautifulSoup(content)\n",
    "\n",
    "        \n",
    "        try:\n",
    "            next_page_button=driver.find_element(By.XPATH,\"//button[contains(@class, 'artdeco-pagination__button artdeco-pagination__button--next artdeco-button artdeco-button--muted artdeco-button--icon-right artdeco-button--1 artdeco-button--tertiary ember-view') and contains(., 'Siguiente')]\")        \n",
    "            next_page_button.click()\n",
    "        except:\n",
    "            print(\"error\")\n",
    "\n",
    "\n",
    "def goto_messages(driver):\n",
    "    try:\n",
    "        messages=driver.find_element(By.LINK_TEXT, \"Mensajes\")\n",
    "        messages.click()\n",
    "    except:\n",
    "        messages=driver.find_element(By.XPATH, \"//*[@id='global-nav']/div/nav/ul/li[4]/a/span\")\n",
    "        messages.click()\n",
    "\n",
    "\n",
    "def get_content(driver):\n",
    "    content = driver.page_source\n",
    "    soup = BeautifulSoup(content, \"html.parser\")\n",
    "    return soup\n",
    "\n",
    "\n",
    "def add_to_exclusion_list(name):\n",
    "    f = open(\"exclusion_list.txt\", \"a\")\n",
    "    f.write(\"\\n\"+name)\n",
    "    f.close()\n",
    "    \n",
    "\n",
    "def get_from_exclusion_list():\n",
    "    exclusion_list=[]\n",
    "    file1 = open('exclusion_list.txt', 'r')\n",
    "    Lines = file1.readlines()\n",
    "\n",
    "    count = 0\n",
    "    # Strips the newline character\n",
    "    for line in Lines:\n",
    "        exclusion_list.append(line.strip())\n",
    "    \n",
    "    return exclusion_list\n",
    "\n",
    "\n",
    "def in_exclusion_list(name):\n",
    "    exclusion_list=[]\n",
    "    file1 = open('exclusion_list.txt', 'r')\n",
    "    Lines = file1.readlines()\n",
    "\n",
    "    count = 0\n",
    "    # Strips the newline character\n",
    "    for line in Lines:\n",
    "        if line.strip()==name:\n",
    "            return True\n",
    "    \n",
    "    return False\n",
    "\n",
    "\n",
    "\n",
    "def del_from_exclusion_list(name):\n",
    "    lines=[]\n",
    "    lines=get_from_exclusion_list() #get all excluded members\n",
    "    lines.remove(name)\n",
    "        \n",
    "    \n",
    "    file = open(\"exclusion_list.txt\", \"w\") #rewrite everything\n",
    "    for line in lines:\n",
    "        file.write(line + \"\\n\")\n",
    "    \n",
    "    file.close()\n",
    "\n",
    "\n",
    "\n",
    "def send_telegram_alert(status, url, name):\n",
    "    bot_message=None\n",
    "    if status==\"ON\":\n",
    "        bot_message=\"🟢 \"+name+\" has activated the BOT again\"\n",
    "    elif status==\"OFF\":\n",
    "        bot_message=\"🔴 \"+name+ \" wants to talk directly with you! \\nReply here: ⬇️ ⬇️ ⬇️\\n\" +url\n",
    "\n",
    "    bot_token = 'xxxxxxxxxxxxxxxxxxxxxxxxx'\n",
    "    bot_chatID = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'\n",
    "    send_text = 'https://api.telegram.org/bot' + bot_token + '/sendMessage?chat_id=' + bot_chatID + '&parse_mode=Markdown&text=' + bot_message\n",
    "\n",
    "    response = requests.get(send_text)\n",
    "\n",
    "    print(response.json())\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def check_bot_status(status, url, name, driver):\n",
    "    \n",
    "    #if any of the messages contains \"start bot\" delete name from the exclusion list\n",
    "    if \"on\" in status.lower():\n",
    "        # Enviar mensaje a telegram\n",
    "        \n",
    "        if in_exclusion_list(name)==True: #user in exclusion list so we can delete it\n",
    "            del_from_exclusion_list(name)\n",
    "            get_content(driver)\n",
    "            send_telegram_alert(\"ON\", url, name)\n",
    "            print(\"__________________________ \"+name+ \" DELETED FROM EXCLUSION LIST __________________________\")\n",
    "        else:\n",
    "            print(\"__________________________ \"+name+ \" ISNT IN EXCLUSION LIST __________________________\")\n",
    "        \n",
    "    #if last message is stop bot, add member to exclusion list\n",
    "    elif \"off\" in status.lower():\n",
    "        # Add name to the exclusion list\n",
    "        \n",
    "        if in_exclusion_list(name)==False: #user not in exclusion list so we can add it\n",
    "            add_to_exclusion_list(name)\n",
    "            get_from_exclusion_list()\n",
    "            get_content(driver)\n",
    "            send_telegram_alert(\"OFF\", url, name)  \n",
    "            print(\"__________________________ \"+name+ \" ADDED TO EXCLUSION LIST __________________________\")\n",
    "        else:\n",
    "            print(\"__________________________ \"+name+ \" ALREADY IN EXCLUSION LIST __________________________\")\n",
    "\n",
    "\n",
    "\n",
    "def reply_private_message(response, driver):\n",
    "\n",
    "    # Getting the reply element                            \n",
    "    reply_box = driver.find_element(By.XPATH, \"//div[@aria-label='Escribe un mensaje…']\") \n",
    "    \n",
    "    print(reply_box)\n",
    "\n",
    "    #reply_box=driver.find_element(By.LINK_TEXT, \"Escribe un mensaje\")\n",
    "    #password_box.clear()\n",
    "    time.sleep(1)\n",
    "    \n",
    "\n",
    "    # Sending the keys for password   \n",
    "    reply_box.send_keys(response) \n",
    "    reply_box.send_keys() \n",
    "    \n",
    "    time.sleep(1)\n",
    "    \n",
    "    send_button=driver.find_element(By.XPATH, \"//button[@type='submit']\") \n",
    "    send_button.click()\n",
    "    \n",
    "    print(\"MESSAGE SENT: \")\n",
    "    print(response)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def language_detection(text):\n",
    "\n",
    "    language=detect_langs(text)\n",
    "    print(language)\n",
    "    first_language=language[0]\n",
    "    \n",
    "    language, score=str(first_language).split(\":\")\n",
    "    return language\n",
    "\n",
    "\n",
    "def add_connect_message(driver):\n",
    "    add_note_button=driver.find_element(By.XPATH,\"//button[contains(@class, 'artdeco-button artdeco-button--muted artdeco-button--2 artdeco-button--secondary ember-view mr1') and contains(., 'Añadir una nota')]\")\n",
    "    #add_note_button.click()\n",
    "                \n",
    "    time.sleep(1)\n",
    "    \n",
    "def send_connect_message(driver):    \n",
    "    # Getting the password element                            \n",
    "    text_box = driver.find_element(By.ID, \"custom-message\") \n",
    "    print(text_box)\n",
    "    text_box.send_keys(config.username)\n",
    "    \n",
    "    enviar_button=driver.find_element(By.XPATH,\"//button[contains(@class, 'artdeco-button artdeco-button--2 artdeco-button--primary ember-view ml1') and contains(., 'Enviar')]\")\n",
    "    enviar_button.click()\n",
    "    print(\"Invitation sent\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5a1b657",
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_number_of_chats():\n",
    "    goto_messages(driver)\n",
    "    # Save all the content in the \"soup\" variable\n",
    "    goto_messages(driver)\n",
    "    soup=get_content(driver)\n",
    "    \n",
    "    # Save all the \"li\" so we can iterate over all the different chats\n",
    "    new = soup.findAll('li', {'class':'scaffold-layout__list-item msg-conversation-listitem msg-conversations-container__convo-item msg-conversations-container__pillar ember-view'})\n",
    "    return len(new)\n",
    "\n",
    "def save_pipeline_list(pipeline_list):\n",
    "    saved_pipeline_list=[]\n",
    "    saved_pipeline_list=pipeline_list\n",
    "    return saved_pipeline_list\n",
    "\n",
    "def initiate_all_pipelines(pipelines_dict):\n",
    "    # iterate over the conversations to\n",
    "    goto_messages(driver)\n",
    "    # Save all the content in the \"soup\" variable\n",
    "    goto_messages(driver)\n",
    "    soup=get_content(driver)\n",
    "    \n",
    "    # empty dictionary\n",
    "\n",
    "    # Save all the \"li\" so we can iterate over all the different chats\n",
    "    new = soup.findAll('li', {'class':'scaffold-layout__list-item msg-conversation-listitem msg-conversations-container__convo-item msg-conversations-container__pillar ember-view'})\n",
    "    for item in new:\n",
    "        name=item.h3.get_text()\n",
    "        name=name.strip()\n",
    "        name=name.replace(\" \", \"_\")\n",
    "        \n",
    "        \n",
    "        if name not in pipelines_dict:\n",
    "            pipeline_name = Model.initiate_conversational_pipeline(name)\n",
    "            pipelines_dict[name] = pipeline_name\n",
    "    \n",
    "    return pipelines_dict\n",
    "\n",
    "\n",
    "\n",
    "def get_messages2(driver, pipelines_dict):\n",
    "    goto_messages(driver)\n",
    "    # Save all the content in the \"soup\" variable\n",
    "    goto_messages(driver)\n",
    "    soup=get_content(driver)\n",
    "    \n",
    "    exclusion_list = []\n",
    "    previus_comment=\"\"\n",
    "    \n",
    "    # Save all the \"li\" so we can iterate over all the different chats\n",
    "    new = soup.findAll('li', {'class':'scaffold-layout__list-item msg-conversation-listitem msg-conversations-container__convo-item msg-conversations-container__pillar ember-view'})\n",
    "    for item in new: \n",
    "        \n",
    "        initiate_pipeline=0\n",
    "        \n",
    "        # Every item is a different chat with a different person\n",
    "        name=item.h3.get_text()\n",
    "        name=name.strip()\n",
    "        print(\" ########################### CONVERSATION WITH \"+name+\" #########################\")\n",
    "        if name not in pipelines_dict: # There isnt conversational pipeline for this person\n",
    "            pipeline_list=initiate_all_pipelines(pipelines_dict)\n",
    "            \n",
    "        # Dynamically created Xpath    \n",
    "        href_text=\"//h3[normalize-space()=\"+str(\"\\'\")+name+str(\"\\'\")+str(\"]\")\n",
    "           \n",
    "        #Based on the href reference go to the private conversation\n",
    "        try:\n",
    "            show_message_button=driver.find_element(By.XPATH, href_text)        \n",
    "            show_message_button.click()\n",
    "            time.sleep(1)\n",
    "        except:\n",
    "            print(\"Error: No more elements\")\n",
    "\n",
    "        time.sleep(2)\n",
    "        soup=get_content(driver)\n",
    "        \n",
    "        name_div=None\n",
    "        send_message=0    \n",
    "\n",
    "        #if \"quick replies\" div exists, means that i have to answer\n",
    "        name_div=soup.find('div', {'class':'conversations-quick-replies justify-center display-flex align-items-center'})\n",
    "\n",
    "        if name_div is None:\n",
    "            print(\"__________________________ SKIP THIS CONVER, LAST MESSAGE IS ALREADY MINE __________________________\")\n",
    "        else:\n",
    "            print(\"__________________________ I HAVE TO RESPOND __________________________\")\n",
    "            send_message=1\n",
    "                \n",
    "        slide_message=\"\"\n",
    "        whole_message=\"\"\n",
    "        \n",
    "        # If im in this loop, at least the last message must be fron the other person\n",
    "        if send_message==1: #Last message isnt mine, i have to respond\n",
    "        # First, analize the \"slide in\" div   \n",
    "            \n",
    "            # Get the url to send to the telegram, if the user types \"stop bot\" i will receive a message with the url \n",
    "            # si i can answer it via linkedin app in the mobile\n",
    "            url=driver.current_url\n",
    "        \n",
    "            result=0\n",
    "            \n",
    "            try:\n",
    "                print(\"__________________________ SLIDE IN (CSS protected div) __________________________\")\n",
    "                result=None\n",
    "                result=soup.find('li',{'class':'msg-s-message-list__event clearfix msg-s-message-list__event--slide-in'})\n",
    "\n",
    "                slide_message=result.find('p',{'class':'msg-s-event-listitem__body t-14 t-black--light t-normal'}).get_text()\n",
    "                print(slide_message)\n",
    "                \n",
    "                result=check_status(slide_message, url, name, driver)\n",
    "                \n",
    "                if result==False:  # If last message isnt stop/start, save it\n",
    "                    print(\"Normal message\")\n",
    "                    slide_message=slide_message+str(\".|.\")\n",
    "                    \n",
    "                elif result==2:    # last message is stop\n",
    "                    print(\"STOP MESSAGE\")\n",
    "                else:              # last message is start\n",
    "                    print(\"START MESSAGE\")\n",
    "                    \n",
    "            except:\n",
    "                print(\"cannot get slide in message (probably is a gif, emoji...)\")\n",
    "            \n",
    "\n",
    "            try:\n",
    "                message_sender=result.find(\"a\",{'class':'msg-s-message-group__profile-link ember-view'}).get_text().strip()\n",
    "                print(\"Message sent by: \"+message_sender)\n",
    "               \n",
    "                # Save it\n",
    "                # Is the last message, could be the first one or the others were sent at a different time\n",
    "                #whole_message=str(last_message)+str(\".|.\")\n",
    "\n",
    "                \n",
    "            except:\n",
    "                # Is the last message but the sender sent more\n",
    "                print(\"Message is a continuation\")\n",
    "                #whole_message=str(last_message)+str(\".|.\")\n",
    "\n",
    "            \n",
    "            if result==2 or result==1: #if the last message is start/stop, end we dont need anything\n",
    "                pass\n",
    "            else:\n",
    "                \n",
    "                ### Now its time to analize all the conversation, from bottom to top, if the messages are sent at a different\n",
    "                # time, there will be different divs so we have to analyze them all until we get our name, that means that the \n",
    "                # last message is already analyzed\n",
    "\n",
    "                # this loop iterates over all messages in the chat with a concrete person\n",
    "                # Last message already analized\n",
    "\n",
    "\n",
    "                li_divs=soup.findAll('li', {'class':'msg-s-message-list__event clearfix'})\n",
    "                whole_message=\"\"\n",
    "                for div in reversed(li_divs):\n",
    "                    time.sleep(1)\n",
    "                    soup=get_content(driver)\n",
    "\n",
    "                    message_sender=None\n",
    "\n",
    "                    print(\"__________________________ NORMAL __________________________\")\n",
    "                    try:\n",
    "                        message_sender=div.find(\"a\",{'class':'msg-s-message-group__profile-link ember-view'}).get_text().strip()\n",
    "\n",
    "                        if message_sender != \"Mikel Uranga\":\n",
    "                            print(\"Message sent by: \"+message_sender)\n",
    "\n",
    "                            try:\n",
    "                                normal_message=div.find('p',{'class':'msg-s-event-listitem__body t-14 t-black--light t-normal'}).get_text()\n",
    "                                print(normal_message)\n",
    "                                \n",
    "                                inner_result=check_status(normal_message, url, name, driver)\n",
    "                                if inner_result==False:\n",
    "                                    whole_message=str(normal_message)+str(\".|.\")+whole_message\n",
    "                                elif inner_result==2: #stop!\n",
    "                                    print(\"STOP\")\n",
    "                                    break\n",
    "                                else:\n",
    "                                    print(\"START\")\n",
    "                                    break\n",
    "                                \n",
    "                            except:\n",
    "                                print(\"could get message\")\n",
    "                        else:\n",
    "                            # This means that the sender is me again, so i must save my comment to improve the translation and\n",
    "                            # exit the iteration and stop analyzing\n",
    "                            \n",
    "                            previus_comment=div.find('p',{'class':'msg-s-event-listitem__body t-14 t-black--light t-normal'}).get_text() \n",
    "                            break\n",
    "                            \n",
    "                    except:\n",
    "                        print(\"no message sender, Message is a continuation\")\n",
    "                        try:\n",
    "                            normal_message=div.find('p',{'class':'msg-s-event-listitem__body t-14 t-black--light t-normal'}).get_text()\n",
    "                            inner_result=check_status(normal_message, url, name, driver)\n",
    "                            if inner_result==False:\n",
    "                                whole_message=str(normal_message)+str(\".|\")+whole_message\n",
    "                            elif inner_result==2: #stop!\n",
    "                                print(\"STOP\")\n",
    "                                break\n",
    "                            else:\n",
    "                                print(\"START\")\n",
    "                                break\n",
    "                        except:\n",
    "                            print(\"could get message\")\n",
    "\n",
    "                    print(whole_message+slide_message)\n",
    "\n",
    "                \n",
    "\n",
    "                if whole_message == \"\" and result==1: #Status is start and there is no more messages\n",
    "                    pass \n",
    "                else:\n",
    "                    \n",
    "                    previus_comment=whole_message\n",
    "\n",
    "                # get the users in the exclusion list (those who asked for a manual conversation)\n",
    "                    exclusion_list=get_from_exclusion_list()\n",
    "                    \n",
    "                    last_message=whole_message+slide_message\n",
    "\n",
    "                    if name not in exclusion_list:\n",
    "                        print(\"__________________________ ULTIMO MENSAJE RECIBIDO  __________________________\")\n",
    "                        print(last_message)\n",
    "\n",
    "                        # Get laguage\n",
    "                        language=language_detection(last_message)\n",
    "\n",
    "                        #if the text is in spanish, translate to english and send to the model (model works in english)\n",
    "                        #then translate the reply again to spanish\n",
    "                        \n",
    "                        # Get value of pipeline for the given name\n",
    "                        pipeline_key=name.replace(\" \", \"_\")\n",
    "                        pipeline=pipeline_list.get(pipeline_key)\n",
    "\n",
    "                        if language==\"en\":\n",
    "                            # Delete all the \"|\" and send directly to model\n",
    "\n",
    "                            #translating\n",
    "                            print(\"Sending message to model\")\n",
    "                            #last_message=last_message.replace(\"|\", \"\")\n",
    "\n",
    "                            response=None\n",
    "                            response = Model.generate_response(last_message, pipeline)\n",
    "                            print(\"Response is\"+response)\n",
    "\n",
    "                            reply_private_message(response, driver)\n",
    "\n",
    "                        else:\n",
    "\n",
    "\n",
    "                            # ignore start bot and stop bot\n",
    "                            print(\"translating from spanish\")\n",
    "                            eng_response=None\n",
    "                            eng_response=Model.translate_es_en(last_message)\n",
    "\n",
    "                            print(eng_response)\n",
    "\n",
    "                            english_response=None\n",
    "                            english_response = ''.join(eng_response)\n",
    "                            \n",
    "                            # This is the question\n",
    "                            print(english_response)\n",
    "\n",
    "                            response=None\n",
    "                            print(\"Sending message to model\")\n",
    "                            \n",
    "                            # This is the response\n",
    "                            response = Model.generate_response(english_response, pipeline)\n",
    "                            \n",
    "                            print(response)\n",
    "                            response=response.replace(\"|\", \"\")\n",
    "\n",
    "                            esp_response=None\n",
    "                            print(\"Translating to spanish again\")\n",
    "                            #Send question an response, so the context is better and it will translate better too\n",
    "\n",
    "                            # /#/ used to divide the question and the response, other type of delimitation characters are deleted\n",
    "                            # by the translation model\n",
    "                            esp_response=Model.translate_en_es(english_response+\"/#/\"+response)\n",
    "                            esp_response_splitted=esp_response[0].split(\"/#/\")\n",
    "    \n",
    "                            #translate result to spanish\n",
    "                            reply_private_message(esp_response_splitted[1], driver)\n",
    "            \n",
    "        \n",
    "\n",
    "            \n",
    "def check_status(last_message, url, name, driver):\n",
    "    # Return True if message is either start bot/stop bot and false if is any other thing\n",
    "    if \"start\" in last_message.lower() and \"bot\" in last_message.lower():\n",
    "        check_bot_status(\"ON\", url, name, driver)\n",
    "        print(\"MESSAGE IS START\")\n",
    "        return 1\n",
    "        \n",
    "    elif \"stop\" in last_message.lower() and \"bot\" in last_message.lower():\n",
    "        check_bot_status(\"OFF\", url, name, driver)\n",
    "        print(\"MESSAGE IS STOP\")\n",
    "        return 2\n",
    "    \n",
    "    else: # No stop/start, then get message to translate\n",
    "        print(\"NO START/STOP\")\n",
    "        return False\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69eb579e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "030e95a8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}