automation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import string\n",
    "import os.path\n",
    "from fuzzywuzzy import fuzz\n",
    "from transformers import pipeline\n",
    "from transformers import AutoTokenizer, AutoModelForTokenClassification\n",
    "import spacy\n",
    "from spacy.matcher import PhraseMatcher\n",
    "from spacy.pipeline import EntityRecognizer\n",
    "from spacy.training import Corpus\n",
    "from spacy import displacy\n",
    "from spacy.tokens import Doc\n",
    "from thinc.api import Config\n",
    "from spacy.language import Language\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import urllib.parse\n",
    "from lxml import etree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "omit_in_search_str = {\"UNITED STATES OF AMERICA\", \"MISC GENERAL MERCHANDISE\", \"VIDEO AMUSEMENT GAME SUPPLY\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "api_url = 'https://app.scrapingbee.com/api/v1/?'\n",
    "api_key_old1 = 'PFAX0VZTO4EPTYIHED58F8GZ5E298GQWMETO6X6BO51Y43QQV80H1EFIP6J6EN7NKLJSBG5L6T2TZRU4'\n",
    "api_key_old2 = 'ZST8Y9KSLMSRUANUZL7D31PW7ND24HPNPBUAPSU1BEXV835ZXUISY2D929FI8UQ55KA7MCUFM1GRCT75'\n",
    "api_key = 'GG64U7ORER3EJ76EP7UQ5HS5TGKYA8YDTF2PAELH5N96LXDI49QPT0I1BQJ39U96ALIW1SFCU2KBQLIU'\n",
    "ADDRESS = 0\n",
    "BIZ_NAME = 1\n",
    "UNWANTED_PHRASE = 2\n",
    "TEST_DATA_PATH = \"C:\\Smrithi\\Internships\\Algotrak\\algotrak-automation\\TestDataSmrithi.xlsx\"\n",
    "TEST_DATA_PATH_REL = \".\\..\\datasets\\TestDataSmrithi.xlsx\"\n",
    "\n",
    "cardinal = '(\\d+[:]CARDINAL[;])'\n",
    "fac = '(\\d+[:]FAC[;])'\n",
    "gpe = '(\\d+[:]GPE[;])'\n",
    "loc = '(\\d+[:]LOC[;])'\n",
    "ordinal = '(\\d+[:]ORDINAL[;])'\n",
    "org = '(\\d+[:]ORG[;])'\n",
    "product = '(\\d+[:]PRODUCT[;])'\n",
    "quantity = '(\\d+[:]QUANTITY[;])'\n",
    "work_of_art = '(\\d+[:]WORK_OF_ART[;])'\n",
    "date = '(\\d+[:]DATE[;])'\n",
    "punct = '(\\d+[:]PUNCT[;])'\n",
    "noent = '(\\d+[:]NOENT[;])'\n",
    "par_open = '('\n",
    "par_close = ')'\n",
    "or_pipe = '|'\n",
    "star = '*'\n",
    "plus = '+'\n",
    "quest_mark = '?'\n",
    "dollar = '$'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "addr_pattern_pt1 = cardinal + star + fac + star + punct + star + loc + star + punct + star + ordinal + star\n",
    "addr_pattern_pt2 = punct + star + par_open + fac + star + punct + star + par_close + star + gpe + plus + punct + star\n",
    "addr_pattern_pt3 = par_open + gpe + star + punct + star + par_close + star + cardinal + star + punct + star + gpe + star\n",
    "addr_pattern = addr_pattern_pt1 + addr_pattern_pt2 + addr_pattern_pt3\n",
    "biz_name_pattern = par_open + product + or_pipe + work_of_art + par_close + star + org + plus\n",
    "unwanted_phrase_pattern = par_open + product + or_pipe + work_of_art + or_pipe + noent + or_pipe + punct + par_close + plus + dollar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Init</th>\n",
       "      <th>Gold Name</th>\n",
       "      <th>Gold Address</th>\n",
       "      <th>Match Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HUA FU FAT BULL SEAFOO RIVERSIDE CA 92507 UNIT...</td>\n",
       "      <td>92507</td>\n",
       "      <td>RIVERSIDE, CA 92507, USA</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>KARZU INVESTMENT, LLC PHARR TX 78577 UNITED ST...</td>\n",
       "      <td>PHARR</td>\n",
       "      <td>PHARR, TX 78577, USA</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14 RBRT GAHANNA COLUMBUS OH 43230 UNITED STATE...</td>\n",
       "      <td>43230</td>\n",
       "      <td>GAHANNA, OH 43230, USA</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>176 COMP KITCH 877-280-73 TROY MI 48084 UNITED...</td>\n",
       "      <td>48084</td>\n",
       "      <td>TROY, MI 48084, USA</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4TH STREET PARKING BERKELEY CA 94710 UNITED ST...</td>\n",
       "      <td>4TH STREET PARKING, LLC</td>\n",
       "      <td>BERKELEY, CA 94710, USA</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>USAVEND AT AIR SERV SUGAR HILL GA 30518 UNITED...</td>\n",
       "      <td>30518</td>\n",
       "      <td>SUGAR HILL, GA 30518, USA</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>USAVEND AT COCA-COLA DALLAS NC 28034 UNITED ST...</td>\n",
       "      <td>DALLAS</td>\n",
       "      <td>DALLAS, NC, USA</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>VENDING BRANDON ROBB DES MOINES IA 50310 UNITE...</td>\n",
       "      <td>50310</td>\n",
       "      <td>DES MOINES, IA 50310, USA</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>VENDING CT VENDING CASTLE PINES CO 80108 UNITE...</td>\n",
       "      <td>CASTLE PINES</td>\n",
       "      <td>CASTLE PINES, CO 80108, USA</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>VENDING EDWARD MCINNIS MI FREELAND MI 48623 UN...</td>\n",
       "      <td>FREELAND</td>\n",
       "      <td>FREELAND, MI 48623, USA</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 Init  \\\n",
       "0   HUA FU FAT BULL SEAFOO RIVERSIDE CA 92507 UNIT...   \n",
       "1   KARZU INVESTMENT, LLC PHARR TX 78577 UNITED ST...   \n",
       "2   14 RBRT GAHANNA COLUMBUS OH 43230 UNITED STATE...   \n",
       "3   176 COMP KITCH 877-280-73 TROY MI 48084 UNITED...   \n",
       "4   4TH STREET PARKING BERKELEY CA 94710 UNITED ST...   \n",
       "..                                                ...   \n",
       "95  USAVEND AT AIR SERV SUGAR HILL GA 30518 UNITED...   \n",
       "96  USAVEND AT COCA-COLA DALLAS NC 28034 UNITED ST...   \n",
       "97  VENDING BRANDON ROBB DES MOINES IA 50310 UNITE...   \n",
       "98  VENDING CT VENDING CASTLE PINES CO 80108 UNITE...   \n",
       "99  VENDING EDWARD MCINNIS MI FREELAND MI 48623 UN...   \n",
       "\n",
       "                  Gold Name                 Gold Address  Match Type  \n",
       "0                     92507     RIVERSIDE, CA 92507, USA        2000  \n",
       "1                     PHARR         PHARR, TX 78577, USA        2000  \n",
       "2                     43230       GAHANNA, OH 43230, USA        2000  \n",
       "3                     48084          TROY, MI 48084, USA        2000  \n",
       "4   4TH STREET PARKING, LLC      BERKELEY, CA 94710, USA        2000  \n",
       "..                      ...                          ...         ...  \n",
       "95                    30518    SUGAR HILL, GA 30518, USA        2000  \n",
       "96                   DALLAS              DALLAS, NC, USA        2000  \n",
       "97                    50310    DES MOINES, IA 50310, USA        2000  \n",
       "98             CASTLE PINES  CASTLE PINES, CO 80108, USA        2000  \n",
       "99                 FREELAND      FREELAND, MI 48623, USA        2000  \n",
       "\n",
       "[100 rows x 4 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cols_req = ['Init', 'Gold Name', 'Gold Address', 'Match Type']\n",
    "test_data = pd.read_excel(TEST_DATA_PATH_REL, usecols=cols_req)\n",
    "test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Guam\n"
     ]
    }
   ],
   "source": [
    "state_names = pd.read_csv(\"../datasets/state-name-codes.csv\")\n",
    "state_names.drop('Abbrev', axis=1, inplace=True)\n",
    "state_names_dict = {}\n",
    "for index, row in state_names.iterrows():\n",
    "    state_names_dict[row[\"Code\"]] = row[\"State\"]\n",
    "print(state_names_dict['GU'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ner': 60528.98046875}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cust_ner = spacy.blank(\"en\")\n",
    "ner = cust_ner.add_pipe(\"ner\")\n",
    "train_corpus = Corpus(\"../datasets/train.spacy\")\n",
    "train_data = list(train_corpus(cust_ner))\n",
    "cust_ner.initialize()\n",
    "cust_ner.update(train_data)\n",
    "\n",
    "# ------------------ OR -----------------------------\n",
    "\n",
    "#config = Config().from_disk(\"./config.cfg\")\n",
    "#cust_ner = Language.from_config(config)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GAMESTOP.COM-O-\n",
      "19-O-\n",
      "2700-O-\n",
      "MIA-O-\n",
      "DAYTON-O-\n",
      "OH-O-\n",
      "45459-O-\n",
      "UNITED-O-\n",
      "STATES-O-\n",
      "OF-O-\n",
      "AMERICA-B-location\n",
      "VIDEO-O-\n",
      "AMUSEMENT-O-\n",
      "GAME-O-\n",
      "SUPPLY-O-\n"
     ]
    }
   ],
   "source": [
    "cust_ner = spacy.load(\"./output/model-best\")\n",
    "doc = cust_ner(\"GAMESTOP.COM 19 2700 MIA DAYTON OH 45459 UNITED STATES OF AMERICA VIDEO AMUSEMENT GAME SUPPLY\")\n",
    "for token in doc:\n",
    "    print(token.text + \"-\" + token.ent_iob_ + \"-\" + token.ent_type_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[paths]\n",
      "train = \"../datasets/train.spacy\"\n",
      "dev = \"../datasets/dev.spacy\"\n",
      "vectors = null\n",
      "init_tok2vec = null\n",
      "\n",
      "[system]\n",
      "seed = 0\n",
      "gpu_allocator = null\n",
      "\n",
      "[nlp]\n",
      "lang = \"en\"\n",
      "pipeline = [\"ner\"]\n",
      "disabled = []\n",
      "before_creation = null\n",
      "after_creation = null\n",
      "after_pipeline_creation = null\n",
      "batch_size = 1000\n",
      "tokenizer = {\"@tokenizers\":\"spacy.Tokenizer.v1\"}\n",
      "\n",
      "[components]\n",
      "\n",
      "[components.ner]\n",
      "factory = \"ner\"\n",
      "incorrect_spans_key = null\n",
      "moves = null\n",
      "update_with_oracle_cut_size = 100\n",
      "\n",
      "[components.ner.model]\n",
      "@architectures = \"spacy.TransitionBasedParser.v2\"\n",
      "state_type = \"ner\"\n",
      "extra_state_tokens = false\n",
      "hidden_width = 64\n",
      "maxout_pieces = 2\n",
      "use_upper = true\n",
      "nO = null\n",
      "\n",
      "[components.ner.model.tok2vec]\n",
      "@architectures = \"spacy.HashEmbedCNN.v2\"\n",
      "pretrained_vectors = null\n",
      "width = 96\n",
      "depth = 4\n",
      "embed_size = 2000\n",
      "window_size = 1\n",
      "maxout_pieces = 3\n",
      "subword_features = true\n",
      "\n",
      "[corpora]\n",
      "\n",
      "[corpora.dev]\n",
      "@readers = \"spacy.Corpus.v1\"\n",
      "path = ${paths.dev}\n",
      "gold_preproc = false\n",
      "max_length = 0\n",
      "limit = 0\n",
      "augmenter = null\n",
      "\n",
      "[corpora.train]\n",
      "@readers = \"spacy.Corpus.v1\"\n",
      "path = ${paths.train}\n",
      "gold_preproc = false\n",
      "max_length = 0\n",
      "limit = 0\n",
      "augmenter = null\n",
      "\n",
      "[training]\n",
      "seed = ${system.seed}\n",
      "gpu_allocator = ${system.gpu_allocator}\n",
      "dropout = 0.1\n",
      "accumulate_gradient = 1\n",
      "patience = 1600\n",
      "max_epochs = 0\n",
      "max_steps = 20000\n",
      "eval_frequency = 200\n",
      "frozen_components = []\n",
      "annotating_components = []\n",
      "dev_corpus = \"corpora.dev\"\n",
      "train_corpus = \"corpora.train\"\n",
      "before_to_disk = null\n",
      "\n",
      "[training.batcher]\n",
      "@batchers = \"spacy.batch_by_words.v1\"\n",
      "discard_oversize = false\n",
      "tolerance = 0.2\n",
      "get_length = null\n",
      "\n",
      "[training.batcher.size]\n",
      "@schedules = \"compounding.v1\"\n",
      "start = 100\n",
      "stop = 1000\n",
      "compound = 1.001\n",
      "t = 0.0\n",
      "\n",
      "[training.logger]\n",
      "@loggers = \"spacy.ConsoleLogger.v1\"\n",
      "progress_bar = false\n",
      "\n",
      "[training.optimizer]\n",
      "@optimizers = \"Adam.v1\"\n",
      "beta1 = 0.9\n",
      "beta2 = 0.999\n",
      "L2_is_weight_decay = true\n",
      "L2 = 0.01\n",
      "grad_clip = 1.0\n",
      "use_averages = false\n",
      "eps = 0.00000001\n",
      "learn_rate = 0.001\n",
      "\n",
      "[training.score_weights]\n",
      "ents_f = 1.0\n",
      "ents_p = 0.0\n",
      "ents_r = 0.0\n",
      "ents_per_type = null\n",
      "\n",
      "[pretraining]\n",
      "\n",
      "[initialize]\n",
      "vectors = ${paths.vectors}\n",
      "init_tok2vec = ${paths.init_tok2vec}\n",
      "vocab_data = null\n",
      "lookups = null\n",
      "before_init = null\n",
      "after_init = null\n",
      "\n",
      "[initialize.components]\n",
      "\n",
      "[initialize.tokenizer]\n"
     ]
    }
   ],
   "source": [
    "print(cust_ner.config.to_str())\n",
    "cust_ner.config.to_disk(\"./config.cfg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def checkGoldOutput(nlp, test_data):\n",
    "    test_data.replace(np.nan, '', regex=True, inplace=True) # in-place modification of test_data\n",
    "    init_inputs_series = test_data.loc[:, 'Init'] # Values of Init column alone, as pandas series\n",
    "    init_inputs_list = init_inputs_series.to_list() # init column values series converted to list\n",
    "    init_input_docs = list(nlp.pipe(init_inputs_list)) # list of init column values as docs\n",
    "    matcher = PhraseMatcher(nlp.vocab) # spacy phrase matcher instance\n",
    "\n",
    "    # I will be using the substring 'united states of america'...\n",
    "    # ...to deduce where the business name ends and address starts\n",
    "    matcher.add(\"USA\", [nlp(\"UNITED STATES OF AMERICA\")])\n",
    "    \n",
    "    # Correct names and correct addresses are basically business names and addresses...\n",
    "    # ...deduced from the init strings. I will be storing them in separate lists\n",
    "    correct_names = []\n",
    "    correct_addrs = []\n",
    "    \n",
    "    for doc in init_input_docs: # iterating thru init string docs\n",
    "        matches = matcher(doc, as_spans=True)\n",
    "        if len(matches) > 0: # because if this condition fails, a lotta errors are thrown\n",
    "            \n",
    "            # taking only the first match, in the event of there being...\n",
    "            # ...more than one match\n",
    "            req_match = matches[0]\n",
    "            \n",
    "            # flag to break out of the following while loop\n",
    "            extracted = 0\n",
    "\n",
    "            # offset value to calculate pos of token starting from the matched...\n",
    "            # ...phrase 'united states of america'\n",
    "            i = 1\n",
    "\n",
    "            # this will just be a list of tokens of the address...\n",
    "            # ...substring from init string\n",
    "            inp_addr = []\n",
    "            \n",
    "            # while loop to extract tokens which should contain address data\n",
    "            # the order of tokens extracted will be reversed since we are...\n",
    "            # ...moving backwards from a particular point in the string\n",
    "            while extracted == 0:\n",
    "                token = doc[req_match.start - i]\n",
    "                # the present conditions for considering a token as that...\n",
    "                # ...having address info\n",
    "                if token.is_alpha or token.is_digit or token.is_space or token.is_ascii:\n",
    "                    inp_addr.append(token.text)\n",
    "                    i += 1\n",
    "\n",
    "                # city names like \"new york\" might not make it to the extracted tokens list\n",
    "                # only 'york' will be extracted as the number of tokens extracted...\n",
    "                # ...thus far is also considered\n",
    "                # only way to check is by using info from the iob tagging in the data\n",
    "                if i >= 4 and token.ent_iob_ in ['B', 'O']:\n",
    "                    # we have got what we wanted - the extracted address tokens\n",
    "                    extracted = 1\n",
    "            \n",
    "            # putting the extracted addr tokens in the correct order\n",
    "            inp_addr.reverse()\n",
    "            \n",
    "            # the addr tokens list does not include the 'united states of america'...\n",
    "            # ...phrase which was used to split the string. We thus add...\n",
    "            # ...it to the list, but abbreviated\n",
    "            inp_addr.append('USA')\n",
    "\n",
    "            # so far we have the address substring info. We now get the tokens...\n",
    "            # ...from the start of the string till the last token we extracted\n",
    "            correct_names.append(doc[0: req_match.start - i + 1].text)\n",
    "            \n",
    "            # this is the string which will hold the complete address substring...\n",
    "            # ...constructed using the 'join' function, delimited by space\n",
    "            inp_addr_str = \" \"\n",
    "            correct_addrs.append(inp_addr_str.join(inp_addr))\n",
    "\n",
    "        # there must've been no matches, so we just append a \"---\"...\n",
    "        # ...to each of the extracted-details lists\n",
    "        else:\n",
    "            correct_names.append(\"---\")\n",
    "            correct_addrs.append(\"---\")\n",
    "    \n",
    "    # converting lists to pandas series\n",
    "    correct_name_series = pd.Series(correct_names, name=\"Extracted Name\")\n",
    "    correct_addr_series = pd.Series(correct_addrs, name=\"Extracted Address\")\n",
    "\n",
    "    # getting gold outputs for names and addresses\n",
    "    gold_outputs = test_data.loc[:, ['Gold Name', 'Gold Address']]\n",
    "    \n",
    "    # final results\n",
    "    yes_or_no_list = []\n",
    "    name_comp_list = []\n",
    "    addr_comp_list = []\n",
    "    \n",
    "    # comparing each gold name and addr with extracted (correct) name and addr\n",
    "    # to-do: rewrite this part as a separate function and use pd.series.apply() to perform the ops\n",
    "    for i in range(gold_outputs.shape[0]):\n",
    "        gold_name = str(gold_outputs.at[i, 'Gold Name'])\n",
    "        corr_name = str(correct_name_series.iloc[i])\n",
    "        gold_addr = str(gold_outputs.at[i, 'Gold Address'])\n",
    "        corr_addr = str(correct_addr_series.iloc[i])\n",
    "        name_score = fuzz.token_sort_ratio(gold_name, corr_name)\n",
    "        addr_score = fuzz.token_sort_ratio(gold_addr, corr_addr)\n",
    "        if name_score > 65 and addr_score > 50:\n",
    "            yes_or_no_list.append('Y')\n",
    "        else:\n",
    "            yes_or_no_list.append('N')\n",
    "        name_comp_list.append(name_score)\n",
    "        addr_comp_list.append(addr_score)\n",
    "            \n",
    "    yes_or_no = pd.Series(yes_or_no_list, name=\"Y/N\") \n",
    "    name_comp = pd.Series(name_comp_list, name=\"Name Comparison\") \n",
    "    addr_comp = pd.Series(addr_comp_list, name=\"Address Comparison\")\n",
    "    col_names = [\"Y/N\", \"Extracted Name\", \"Extracted Address\", \"Name Comparison\", \"Address Comparison\"]\n",
    "    return yes_or_no, correct_name_series, correct_addr_series, name_comp, addr_comp # return pandas series\n"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "init_input_docs = nlp.pipe(init_inputs)\n",
    "    patterns = [(biz_name_pattern, BIZ_NAME), (addr_pattern, ADDRESS)]\n",
    "    i = 0\n",
    "    for pattern, pat_flag in patterns:\n",
    "        match_indices = extractPattern(pattern, entityString(init_input_docs), flag=pat_flag)\n",
    "        matches = getMatches(match_indices)\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def send_request(use_params):\n",
    "    response = requests.get(\n",
    "        url=api_url,\n",
    "        params={\n",
    "            'api_key': api_key,\n",
    "            'url': use_params['url'],\n",
    "            'custom_google': use_params['custom_google'],\n",
    "        },        \n",
    "    )\n",
    "    print('Response HTTP Status Code: ', response.status_code)\n",
    "    return(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['GAMESTOP.COM 19 2700 MIA DAYTON Ohio 45459 UNITED STATES OF AMERICA VIDEO AMUSEMENT GAME SUPPLY']\n"
     ]
    }
   ],
   "source": [
    "def modifyString(in_strs):\n",
    "    ret_strs = []\n",
    "    for in_str in in_strs:\n",
    "        ret_str = in_str\n",
    "        for (k, v) in state_names_dict.items():\n",
    "            pattern = \"(?<![a-zA-Z])\" + re.escape(k) + \"(?![a-zA-Z])\"\n",
    "            ret_str = re.sub(pattern, v, ret_str, flags=re.I)\n",
    "        ret_strs.append(ret_str)\n",
    "    return ret_strs\n",
    "\n",
    "stringg = \"MMS MS MS MS907 MSS TX,PA,TN GGGmekgnejbiq\"\n",
    "string2 = \"AAFES GUAM NEW MAIN STOR DALLAS TX 75265 UNITED STATES OF AMERICA MISC GENERAL MERCHANDISE\"\n",
    "string3 = \"GAMESTOP.COM 19 2700 MIA DAYTON OH 45459 UNITED STATES OF AMERICA VIDEO AMUSEMENT GAME SUPPLY\"\n",
    "print(modifyString([string3]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def handleSpecialCases(token):\n",
    "    entity_name = \"NOENT\"\n",
    "    token_text = token.text\n",
    "    if len(token_text) == 1 and token_text in string.punctuation:\n",
    "        entity_name = 'PUNCT'\n",
    "    return entity_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stringInfo(docs):\n",
    "    for doc in docs:\n",
    "        print(doc.text + \"\\n--------------------------------------------------------------------\")\n",
    "        for token in doc:\n",
    "            print(str(token.i))\n",
    "            print(token.text + \": \" + token.ent_iob_ + \"-\" + token.ent_type_ + \" \", end=\"\")\n",
    "            if spacy.explain(token.ent_type_) is not None:\n",
    "                print(spacy.explain(token.ent_type_))\n",
    "            else:\n",
    "                print('No entity')\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['AAFES GUAM NEW MAIN STOR DALLAS TX 75265']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def removeUnwantedPhrases(proc_texts):\n",
    "    ret_proc_texts = proc_texts\n",
    "    for to_omit in omit_in_search_str:\n",
    "        for i in range(len(ret_proc_texts)):\n",
    "            new_str = re.sub(to_omit, \"\", ret_proc_texts[i], flags=re.I)\n",
    "            ret_proc_texts[i] = new_str\n",
    "    ret_proc_texts = [ret_proc_texts[i].strip(string.whitespace + string.punctuation) for i in range(len(ret_proc_texts))]\n",
    "    return ret_proc_texts\n",
    "textx= 'AAFES GUAM NEW MAIN STOR DALLAS TX 75265 UNITED STATES OF AMERICA MISC GENERAL MERCHANDISE'\n",
    "removeUnwantedPhrases([textx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def textPreproc(nlp, texts, dont_remove_unwanted=0):\n",
    "    preproc_texts = modifyString(texts)\n",
    "    if dont_remove_unwanted == 0:\n",
    "        proc_texts = removeUnwantedPhrases(preproc_texts)\n",
    "        preproc_texts = proc_texts\n",
    "    docs = nlp.pipe(proc_texts)\n",
    "    return docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def entityString(docs):\n",
    "    ret_strs = []\n",
    "    for doc in docs:\n",
    "        proc_string = \"\"\n",
    "        for token in doc:\n",
    "            entity_name = \"\"\n",
    "            if len(str(token.ent_type_)) == 0:\n",
    "                entity_name = handleSpecialCases(token)\n",
    "            else:\n",
    "                entity_name = token.ent_type_\n",
    "            add_str = str(token.i) + \":\" + entity_name + \";\"\n",
    "            # print(\"Adding: \" + add_str)\n",
    "            proc_string += add_str\n",
    "        ret_strs.append(proc_string)\n",
    "    return ret_strs # return list of strings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def couldItBeAddr(match):\n",
    "    ents = re.findall('[A-Za-z]+', match.group(0))\n",
    "    ents_dict = {}\n",
    "    ents_dict = dict((ents[i], i)  for i in range(len(ents)) if ents[i] not in ents_dict)\n",
    "    ents_set = set(ents_dict.keys())\n",
    "    if len(ents_set) < 2:\n",
    "        return 0\n",
    "    elif len(ents_set) == 2:\n",
    "        if {\"CARDINAL\", \"GPE\"} == ents_set and ents_dict['CARDINAL'] < ents_dict['GPE']:\n",
    "            return 0\n",
    "        elif \"PUNCT\" in ents_set:\n",
    "            return 0\n",
    "    else:\n",
    "        return 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def couldItBeBizName(match):\n",
    "    return 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def couldItBeUnwantedPhrases(match):\n",
    "    return 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def checkValidity(match, flag):\n",
    "    if flag == ADDRESS:\n",
    "        return couldItBeAddr(match)\n",
    "    elif flag == BIZ_NAME:\n",
    "        return couldItBeBizName(match)\n",
    "    elif flag == UNWANTED_PHRASE:\n",
    "        return couldItBeUnwantedPhrases(match)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extractPattern(pattern, ent_strs, flag=ADDRESS):\n",
    "    full_index_list = []\n",
    "    for ent_str in ent_strs:\n",
    "        index_list = [] # corresponds to one set of matches\n",
    "        matches = re.finditer(pattern, ent_str)\n",
    "        for match in matches:\n",
    "            if checkValidity(match, flag) == 0:\n",
    "                continue\n",
    "            else:\n",
    "                ind = re.findall('\\d+', match.group(0))\n",
    "                indices = list(map(lambda x: int(x), ind))\n",
    "                first_index = indices[0]\n",
    "                last_index = indices[len(indices) - 1] + 1\n",
    "                index_list.append([first_index, last_index]) # [first_index, last_index] corresponds to one match\n",
    "        full_index_list.append(index_list)\n",
    "    return full_index_list # return list of lists of index-pair lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def findUnwantedPhrases(docs):\n",
    "    ents_strs = entityString(docs)\n",
    "    match_indices = extractPattern(unwanted_phrase_pattern, ents_strs, UNWANTED_PHRASE)\n",
    "    match_strings_dict = getMatches(match_indices)\n",
    "    changes_req = bool(match_strings_dict)\n",
    "    if changes_req:\n",
    "        for (k, v) in match_strings_dict:\n",
    "            for match_str in v:\n",
    "                omit_in_search_str.add(match_str.upper())\n",
    "    return changes_req"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getMatches(search_results, orig_strings): # search_results is a list of lists containing match index-pairs\n",
    "    i = 0\n",
    "    matches_acr_src_res = {}\n",
    "    for search_result in search_results:\n",
    "        search_query = orig_strings[i]\n",
    "        matches_in_src_res = []\n",
    "        #print(\"Matches for text: \" + search_query.text)\n",
    "        for text_span in search_result:\n",
    "            final_result = search_query[text_span[0]: text_span[1]].text.strip(string.whitespace + string.punctuation)\n",
    "            #print(final_result)\n",
    "            matches_in_src_res.append(final_result)\n",
    "        matches_acr_src_res[i] = matches_in_src_res\n",
    "        #print(\"------------------------------------------------------------------\\n\")\n",
    "        i += 1\n",
    "    return matches_acr_src_res # return dict with integer keys and list-of-matches values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['en_core_web_trf']\n",
      "['transformer', 'tagger', 'parser', 'attribute_ruler', 'lemmatizer', 'ner']\n"
     ]
    }
   ],
   "source": [
    "print(spacy.util.get_installed_models())\n",
    "\n",
    "nlp = spacy.load(\"en_core_web_trf\")\n",
    "nlp.resume_training()\n",
    "#print(nlp.config.to_str())\n",
    "print(nlp.pipe_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "ret_tuple = checkGoldOutput(nlp, test_data) # y_or_n will hold pandas series\n",
    "if os.path.exists('./output/testdata.xlsx'):\n",
    "    test_data = pd.read_excel('./output/testdata.xlsx')\n",
    "\n",
    "for ret_series in ret_tuple:\n",
    "    if ret_series.name in test_data.columns:\n",
    "        index_loc = test_data.columns.get_loc(ret_series.name)\n",
    "        test_data.drop(ret_series.name, axis=1, inplace=True)\n",
    "        test_data.insert(index_loc, ret_series.name, ret_series)\n",
    "    else:\n",
    "        test_data.insert(test_data.shape[1], ret_series.name, ret_series)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data.to_excel('./output/testdata1.xlsx', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HUA-O-\n",
      "FU-O-\n",
      "FAT-O-\n",
      "BULL-O-\n",
      "SEAFOO-O-\n",
      "RIVERSIDE-O-\n",
      "CA-O-\n",
      "92507-B-CARDINAL\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "KARZU-B-ORG\n",
      "INVESTMENT-I-ORG\n",
      ",-I-ORG\n",
      "LLC-I-ORG\n",
      "PHARR-B-GPE\n",
      "TX-B-GPE\n",
      "78577-B-CARDINAL\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "14-O-\n",
      "RBRT-O-\n",
      "GAHANNA-O-\n",
      "COLUMBUS-B-GPE\n",
      "OH-B-GPE\n",
      "43230-B-CARDINAL\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "176-O-\n",
      "COMP-O-\n",
      "KITCH-O-\n",
      "877-O-\n",
      "--O-\n",
      "280-O-\n",
      "--O-\n",
      "73-O-\n",
      "TROY-O-\n",
      "MI-O-\n",
      "48084-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "FAST-O-\n",
      "FOOD-O-\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "4TH-O-\n",
      "STREET-O-\n",
      "PARKING-B-GPE\n",
      "BERKELEY-B-GPE\n",
      "CA-B-GPE\n",
      "94710-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "PARKING-O-\n",
      "LOTS-O-\n",
      ",-O-\n",
      "METERS-O-\n",
      ",-O-\n",
      "GARAGES-O-\n",
      "-------------------------------------------------------\n",
      "51376-O-\n",
      "--O-\n",
      "115-O-\n",
      "E.87TH-O-\n",
      "STREET-O-\n",
      "NEW-B-GPE\n",
      "YORK-I-GPE\n",
      "NY-O-\n",
      "10128-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "PARKING-O-\n",
      "LOTS-O-\n",
      ",-O-\n",
      "METERS-O-\n",
      ",-O-\n",
      "GARAGES-O-\n",
      "-------------------------------------------------------\n",
      "802-O-\n",
      "LONGHORNS-O-\n",
      "TEAMSHOP-O-\n",
      "CHICAGO-O-\n",
      "IL-O-\n",
      "60614-B-CARDINAL\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "COSMETIC-O-\n",
      "STORES-O-\n",
      "-------------------------------------------------------\n",
      "860-O-\n",
      "LSU-B-ORG\n",
      "TEAM-I-ORG\n",
      "SHOP-O-\n",
      "GREENWICH-B-GPE\n",
      "CT-B-GPE\n",
      "6830-B-CARDINAL\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "COSMETIC-O-\n",
      "STORES-O-\n",
      "-------------------------------------------------------\n",
      "AAFES-B-PERSON\n",
      "GUAM-I-PERSON\n",
      "NEW-O-\n",
      "MAIN-O-\n",
      "STOR-O-\n",
      "DALLAS-B-GPE\n",
      "TX-B-GPE\n",
      "75265-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "MISC-O-\n",
      "GENERAL-O-\n",
      "MERCHANDISE-O-\n",
      "-------------------------------------------------------\n",
      "AAFES-O-\n",
      "KEESLER-O-\n",
      "SHE-O-\n",
      "MCSS-O-\n",
      "HATTIESBURG-B-GPE\n",
      "MS-O-\n",
      "39407-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "MISC-O-\n",
      "GENERAL-O-\n",
      "MERCHANDISE-O-\n",
      "-------------------------------------------------------\n",
      "AAFES-O-\n",
      "MED-O-\n",
      "DET-O-\n",
      "MM-O-\n",
      "TEPORA-O-\n",
      "FORT-O-\n",
      "DETRICK-O-\n",
      "MD-O-\n",
      "21702-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "MISC-O-\n",
      "GENERAL-O-\n",
      "MERCHANDISE-O-\n",
      "-------------------------------------------------------\n",
      "AAFES-O-\n",
      "PAFB-O-\n",
      "ANTHONY-O-\n",
      "S-O-\n",
      "NEW-O-\n",
      "PETERSON-O-\n",
      "AFB-O-\n",
      "CO-O-\n",
      "80914-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "FAST-O-\n",
      "FOOD-O-\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "AAFES-B-ORG\n",
      "--I-ORG\n",
      "JACKSON-I-ORG\n",
      "FC-I-ORG\n",
      "QDOBA-O-\n",
      "FORT-O-\n",
      "JACKSON-O-\n",
      "SC-O-\n",
      "29207-B-CARDINAL\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "FAST-O-\n",
      "FOOD-O-\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "AAFES-O-\n",
      "--O-\n",
      "MCX-O-\n",
      "CAMP-O-\n",
      "FOSTER-O-\n",
      "AR-O-\n",
      "APO-O-\n",
      " -O-\n",
      "96378-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "FAST-O-\n",
      "FOOD-O-\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "AARONS-O-\n",
      "REVRUN-O-\n",
      "F036-O-\n",
      "NEWPORT-O-\n",
      "DE-O-\n",
      "19804-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "EQUIP-O-\n",
      "/-O-\n",
      "FURN-O-\n",
      "RENT-O-\n",
      "/-O-\n",
      "LEASE-O-\n",
      "SERV-O-\n",
      "-------------------------------------------------------\n",
      "AFV-O-\n",
      "447-B-CARDINAL\n",
      "GILBERT-B-GPE\n",
      "AZ-B-GPE\n",
      "85233-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "FAST-O-\n",
      "FOOD-O-\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "AMERICASNATLPARK136J-O-\n",
      "FREDERICKSBRG-O-\n",
      "VA-O-\n",
      "22401-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "GIFT-O-\n",
      ",-O-\n",
      "CARD-O-\n",
      ",-O-\n",
      "NOVELTY-O-\n",
      "STORES-O-\n",
      "-------------------------------------------------------\n",
      "ARAMARK-O-\n",
      "1631-O-\n",
      "SEASONS-O-\n",
      "CLEMSON-B-GPE\n",
      "SC-B-GPE\n",
      "29634-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "FAST-O-\n",
      "FOOD-O-\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "ARAMARK-B-FAC\n",
      "SC-I-FAC\n",
      "SUBWAY-I-FAC\n",
      "LEXINGTON-B-GPE\n",
      "KY-B-GPE\n",
      "40508-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "FAST-O-\n",
      "FOOD-O-\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "ARAMARK-B-PERSON\n",
      "TEMPLETON-I-PERSON\n",
      "DINING-O-\n",
      "MS-O-\n",
      "STATE-O-\n",
      "UNIV-O-\n",
      "MS-O-\n",
      "39762-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "FAST-O-\n",
      "FOOD-O-\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "ASG-O-\n",
      "LOT-O-\n",
      "200-O-\n",
      "JEFFERSON-O-\n",
      "140-O-\n",
      "ARLINGTON-B-GPE\n",
      "VA-B-GPE\n",
      "22202-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "PARKING-O-\n",
      "LOTS-O-\n",
      ",-O-\n",
      "METERS-O-\n",
      ",-O-\n",
      "GARAGES-O-\n",
      "-------------------------------------------------------\n",
      "ATF-B-ORG\n",
      "HOLDINGS-I-ORG\n",
      "LLC-I-ORG\n",
      "SEABROOK-B-GPE\n",
      "ISL-I-GPE\n",
      "SC-I-GPE\n",
      "29455-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "CAR-O-\n",
      "WASHES-O-\n",
      "-------------------------------------------------------\n",
      "51-O-\n",
      "EAST-O-\n",
      "34TH-O-\n",
      "ST-O-\n",
      "DELI-O-\n",
      "NEW-B-GPE\n",
      "YORK-I-GPE\n",
      "NY-O-\n",
      "10016-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "MISC-O-\n",
      "FOOD-O-\n",
      "STORES-O-\n",
      "--O-\n",
      "DEFAULT-O-\n",
      "-------------------------------------------------------\n",
      "72672-O-\n",
      "LAUNDRY-O-\n",
      "CARD-O-\n",
      "BROOKLYN-O-\n",
      "NY-O-\n",
      "11218-O-\n",
      "UNITED-O-\n",
      "STATES-O-\n",
      "OF-O-\n",
      "AMERICA-O-\n",
      "LAUNDRIES-O-\n",
      "--O-\n",
      "FAMILY-O-\n",
      "/-O-\n",
      "COMMERCIAL-O-\n",
      "-------------------------------------------------------\n",
      "76-O-\n",
      "--O-\n",
      "EVELYN-O-\n",
      "AVE-O-\n",
      "76-O-\n",
      "MOUNTAIN-O-\n",
      "VIEW-O-\n",
      "CA-O-\n",
      "94041-O-\n",
      "UNITED-B-ORG\n",
      "STATES-I-ORG\n",
      "OF-I-ORG\n",
      "AMERICA-I-ORG\n",
      "SERVICE-O-\n",
      "STATIONS-O-\n",
      "-------------------------------------------------------\n",
      "98408-O-\n",
      "--O-\n",
      "RT-O-\n",
      "17-O-\n",
      "RAIL-O-\n",
      "PARK-O-\n",
      "/-O-\n",
      "R-O-\n",
      "RAMSEY-O-\n",
      "NJ-B-GPE\n",
      "7446-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "PARKING-O-\n",
      "LOTS-O-\n",
      ",-O-\n",
      "METERS-O-\n",
      ",-O-\n",
      "GARAGES-O-\n",
      "-------------------------------------------------------\n",
      "AAA-O-\n",
      "MIDATL-O-\n",
      "MARLTN-O-\n",
      "Q65103-O-\n",
      "MARLTON-B-GPE\n",
      "NJ-B-GPE\n",
      "8053-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "NON-O-\n",
      "--O-\n",
      "FIN-O-\n",
      "INST-O-\n",
      "/-O-\n",
      "FC-O-\n",
      "/-O-\n",
      "MO-O-\n",
      "/-O-\n",
      "TC-O-\n",
      "-------------------------------------------------------\n",
      "ACCESSORY-O-\n",
      "ST1090-O-\n",
      "NASHVILLE-B-GPE\n",
      "TN-B-GPE\n",
      "37217-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "MISC-O-\n",
      "APPAREL-O-\n",
      "/-O-\n",
      "ACCESS-O-\n",
      "STORES-O-\n",
      "-------------------------------------------------------\n",
      "CAPITAL-O-\n",
      "CITY-O-\n",
      "SMOKE-O-\n",
      "/-O-\n",
      "GRCRY-O-\n",
      "TOPEKA-B-GPE\n",
      "KS-B-GPE\n",
      "66603-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "MISC-O-\n",
      "FOOD-O-\n",
      "STORES-O-\n",
      "--O-\n",
      "DEFAULT-O-\n",
      "-------------------------------------------------------\n",
      "EZ-O-\n",
      "SHOP-O-\n",
      "OF-O-\n",
      "BRONX-O-\n",
      "INC-O-\n",
      "BRONX-B-GPE\n",
      "NY-O-\n",
      "10468-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "GROCERY-O-\n",
      "STORES-O-\n",
      "/-O-\n",
      "SUPERMARKETS-O-\n",
      "-------------------------------------------------------\n",
      "FRISCO-B-FAC\n",
      "LAKES-I-FAC\n",
      "GOLF-I-FAC\n",
      "CLUB-I-FAC\n",
      "WESTLAKEA-B-GPE\n",
      "TX-B-GPE\n",
      "76262-B-CARDINAL\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "PUBLIC-O-\n",
      "GOLF-O-\n",
      "COURSES-O-\n",
      "-------------------------------------------------------\n",
      "GAMESTOP.COM-O-\n",
      "19-O-\n",
      "2700-O-\n",
      "MIA-O-\n",
      "DAYTON-O-\n",
      "OH-B-GPE\n",
      "45459-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "VIDEO-O-\n",
      "AMUSEMENT-O-\n",
      "GAME-O-\n",
      "SUPPLY-O-\n",
      "-------------------------------------------------------\n",
      "HAWAIIAN-O-\n",
      "TIME-O-\n",
      "SPARKS-O-\n",
      "NV-O-\n",
      "89431-B-CARDINAL\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "FAST-O-\n",
      "FOOD-O-\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "HOLLISTER-O-\n",
      "186-O-\n",
      "4841-O-\n",
      "N-O-\n",
      "1ST-O-\n",
      "FRESNO-O-\n",
      "CA-O-\n",
      "93720-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "MENS-O-\n",
      "/-O-\n",
      "WOMENS-O-\n",
      "CLOTHING-O-\n",
      "STORES-O-\n",
      "-------------------------------------------------------\n",
      "IDENTOGO-O-\n",
      "--O-\n",
      "PA-O-\n",
      "FINGERPRINT-O-\n",
      "877-O-\n",
      "--O-\n",
      "512-O-\n",
      "--O-\n",
      "6962-O-\n",
      "PA-O-\n",
      "17013-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "GOV'T-O-\n",
      "SERV-O-\n",
      "--O-\n",
      "DEFAULT-O-\n",
      "-------------------------------------------------------\n",
      "LAKESHORE-B-ORG\n",
      "SEAFOOD-I-ORG\n",
      "&-I-ORG\n",
      "GR-I-ORG\n",
      "985-O-\n",
      "--O-\n",
      "7780326-O-\n",
      "LA-B-GPE\n",
      "70471-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "AAFES-O-\n",
      "BUTLER-O-\n",
      "THEATER-O-\n",
      "APO-O-\n",
      "AP-O-\n",
      "96378-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "MISC-O-\n",
      "GENERAL-O-\n",
      "MERCHANDISE-O-\n",
      "-------------------------------------------------------\n",
      "AAFES-O-\n",
      "FOSTER-O-\n",
      "FC-O-\n",
      "TACO-O-\n",
      "B-O-\n",
      "APO-O-\n",
      "AP-O-\n",
      "96378-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "MISC-O-\n",
      "GENERAL-O-\n",
      "MERCHANDISE-O-\n",
      "-------------------------------------------------------\n",
      "AAFES-O-\n",
      "KADENA-O-\n",
      "BXFC-O-\n",
      "M-O-\n",
      "WO-O-\n",
      "APO-O-\n",
      "AP-O-\n",
      "96378-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "FAST-O-\n",
      "FOOD-O-\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "BART-O-\n",
      "--O-\n",
      "CLIPPER-O-\n",
      "POWELL-O-\n",
      " -O-\n",
      "CA-O-\n",
      " -O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "LOCAL-O-\n",
      "COMMUTER-O-\n",
      "TRANSPORT-O-\n",
      "-------------------------------------------------------\n",
      "BIG-O-\n",
      "Y-O-\n",
      "22-O-\n",
      "16BZ-O-\n",
      " -O-\n",
      "MA-O-\n",
      " -O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "WIRE-O-\n",
      "TRANSFER-O-\n",
      "MONEY-O-\n",
      "ORDER-O-\n",
      "-------------------------------------------------------\n",
      "BROADWAY-O-\n",
      "INN-O-\n",
      "&-O-\n",
      "SUITE-O-\n",
      "4058481919-O-\n",
      "OK-O-\n",
      "0-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "BEST-O-\n",
      "WESTERN-O-\n",
      "-------------------------------------------------------\n",
      "CENTRAARCHY-O-\n",
      "ON-O-\n",
      "LINE-O-\n",
      "843-B-CARDINAL\n",
      "--I-CARDINAL\n",
      "571-O-\n",
      "--O-\n",
      "0096-O-\n",
      "SC-O-\n",
      "0-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "EB-O-\n",
      "TOUGH-O-\n",
      "MUDDER-O-\n",
      "TAHOE-O-\n",
      "801-O-\n",
      "--O-\n",
      "413-O-\n",
      "--O-\n",
      "7200-O-\n",
      "CA-O-\n",
      "0-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "BUSINESS-O-\n",
      "SERVICES-O-\n",
      "--O-\n",
      "DEFAULT-O-\n",
      "-------------------------------------------------------\n",
      "HAWTHORNE-O-\n",
      "SUITES-O-\n",
      " -O-\n",
      "FL-B-GPE\n",
      " -O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "HOTELS-O-\n",
      "/-O-\n",
      "MOTELS-O-\n",
      "/-O-\n",
      "RESORTS-O-\n",
      "-------------------------------------------------------\n",
      "HOLLISTER-O-\n",
      "261-O-\n",
      " -O-\n",
      "CA-O-\n",
      " -O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "MENS-O-\n",
      "/-O-\n",
      "WOMENS-O-\n",
      "CLOTHING-O-\n",
      "STORES-O-\n",
      "-------------------------------------------------------\n",
      "INGLES-O-\n",
      "8-O-\n",
      "PCSM-O-\n",
      " -O-\n",
      "NC-O-\n",
      " -O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "WIRE-O-\n",
      "TRANSFER-O-\n",
      "MONEY-O-\n",
      "ORDER-O-\n",
      "-------------------------------------------------------\n",
      "JARED-O-\n",
      "GALLERIA-O-\n",
      "12430-O-\n",
      "FOOT-O-\n",
      " -O-\n",
      "CA-O-\n",
      " -O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "JEWELRY-O-\n",
      "STORES-O-\n",
      "-------------------------------------------------------\n",
      "LOWES-O-\n",
      "02945-O-\n",
      " -O-\n",
      "KY-O-\n",
      " -O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "HOME-O-\n",
      "SUPPLY-O-\n",
      "WAREHOUSE-O-\n",
      "STORES-O-\n",
      "-------------------------------------------------------\n",
      "MCDONALD-B-ORG\n",
      "'S-I-ORG\n",
      "GITMO-I-ORG\n",
      "BASE-I-ORG\n",
      "USA-I-ORG\n",
      "AE-O-\n",
      "9593-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "FAST-O-\n",
      "FOOD-O-\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "PILOT-O-\n",
      " -O-\n",
      "CA-O-\n",
      " -O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "AUTOMATED-O-\n",
      "FUEL-O-\n",
      "DISPENSERS-O-\n",
      "-------------------------------------------------------\n",
      "SHOWCLIX-O-\n",
      "INC-O-\n",
      "8887184253-O-\n",
      "PA-O-\n",
      "0-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "BUSINESS-O-\n",
      "SERVICES-O-\n",
      "--O-\n",
      "DEFAULT-O-\n",
      "-------------------------------------------------------\n",
      "SMITHS-O-\n",
      "FOOD-O-\n",
      " -O-\n",
      "4351-O-\n",
      "   -O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "GROCERY-O-\n",
      "STORES-O-\n",
      "/-O-\n",
      "SUPERMARKETS-O-\n",
      "-------------------------------------------------------\n",
      "SP-O-\n",
      " -O-\n",
      "INDY-O-\n",
      "+-O-\n",
      "RAE-O-\n",
      "HTTPSINDYRAE-O-\n",
      ".-O-\n",
      "UT-O-\n",
      "84043-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "MENS-O-\n",
      "/-O-\n",
      "WOMENS-O-\n",
      "CLOTHING-O-\n",
      "STORES-O-\n",
      "-------------------------------------------------------\n",
      "SUBWAY-O-\n",
      " -O-\n",
      "NC-B-GPE\n",
      " -O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "FAST-O-\n",
      "FOOD-O-\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "USAA-O-\n",
      "SUBROGATION-O-\n",
      " -O-\n",
      "TX-B-GPE\n",
      " -O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "INSURANCE-O-\n",
      "SALES-O-\n",
      "/-O-\n",
      "UNDERWRITE-O-\n",
      "-------------------------------------------------------\n",
      "ZUMIEZ600-O-\n",
      "CYB-O-\n",
      " -O-\n",
      "WA-O-\n",
      " -O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "FAMILY-O-\n",
      "CLOTHING-O-\n",
      "STORES-O-\n",
      "-------------------------------------------------------\n",
      "AAFES-O-\n",
      "KINSER-O-\n",
      "SHOPETTE-O-\n",
      "APO-O-\n",
      " -O-\n",
      "96378-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "MISC-O-\n",
      "GENERAL-O-\n",
      "MERCHANDISE-O-\n",
      "-------------------------------------------------------\n",
      "CASH-O-\n",
      "AND-O-\n",
      "CARRY-O-\n",
      "ABC-O-\n",
      "BUILDI-O-\n",
      "TAMUNING-O-\n",
      "OH-B-GPE\n",
      "0-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "COMMERCIAL-O-\n",
      "EQUIPMENT-O-\n",
      "--O-\n",
      "DEFAULT-O-\n",
      "-------------------------------------------------------\n",
      "CVS-O-\n",
      "/-O-\n",
      "PHARMACY-O-\n",
      "06-O-\n",
      "06507-O-\n",
      "--O-\n",
      "-W-O-\n",
      " -O-\n",
      "MN-O-\n",
      " -O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "DRUG-O-\n",
      "STORES-O-\n",
      "&-O-\n",
      "PHARMACIES-O-\n",
      "-------------------------------------------------------\n",
      "GCAHOLLYWOOD-O-\n",
      "ST-O-\n",
      " -O-\n",
      "MO-B-GPE\n",
      " -O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "WIRE-O-\n",
      "TRANSFER-O-\n",
      "MONEY-O-\n",
      "ORDER-O-\n",
      "-------------------------------------------------------\n",
      "MEM-O-\n",
      "TAXI-O-\n",
      "514-O-\n",
      " -O-\n",
      "TN-B-GPE\n",
      " -O-\n",
      "-------------------------------------------------------\n",
      "SP-O-\n",
      " -O-\n",
      "CHARGEDD-O-\n",
      "HTTPSCHARGEDE-O-\n",
      "MO-O-\n",
      "65201-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "ELECTRONICS-O-\n",
      "STORES-O-\n",
      "-------------------------------------------------------\n",
      "SPRINT-O-\n",
      "RETAIL-O-\n",
      "253073-O-\n",
      " -O-\n",
      "CA-O-\n",
      " -O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "TELECOMMUNICATION-O-\n",
      "SERVICES-O-\n",
      "-------------------------------------------------------\n",
      "UBER-B-GPE\n",
      " -O-\n",
      "CA-B-GPE\n",
      " -O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "TAXICABS-O-\n",
      "/-O-\n",
      "LIMOUSINES-O-\n",
      "-------------------------------------------------------\n",
      "USAGH-B-ORG\n",
      "MORNING-I-ORG\n",
      "CALM-I-ORG\n",
      "APO-O-\n",
      "AP-O-\n",
      "96271-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "RECREATION-O-\n",
      "SERVICES-O-\n",
      "-------------------------------------------------------\n",
      "USAGH-O-\n",
      "WZ-O-\n",
      "SPT030-O-\n",
      "APO-O-\n",
      "AP-O-\n",
      "96271-B-CARDINAL\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "VERIZON-O-\n",
      "CAFE-O-\n",
      "18092171-O-\n",
      " -O-\n",
      "CO-O-\n",
      " -O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "FAST-O-\n",
      "FOOD-O-\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "AAA-O-\n",
      "TW-O-\n",
      "MEMBER-O-\n",
      "PAYMENT-O-\n",
      "COSTA-O-\n",
      "MESA-B-GPE\n",
      "CA-O-\n",
      "92626-O-\n",
      "UNITED-B-ORG\n",
      "STATES-I-ORG\n",
      "OF-I-ORG\n",
      "AMERICA-I-ORG\n",
      "AUTO-O-\n",
      "ASSOCIATIONS-O-\n",
      "-------------------------------------------------------\n",
      "AADA-O-\n",
      "RW3-O-\n",
      "VANCOUVER-O-\n",
      "WA-O-\n",
      "98686-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "DOCTORS-O-\n",
      "&-O-\n",
      "PHYSICIANS-O-\n",
      "-------------------------------------------------------\n",
      "AAFES-O-\n",
      "JBSA-B-PERSON\n",
      "LACK-I-PERSON\n",
      "ANTHONY-O-\n",
      "LACKLAND-B-GPE\n",
      "AFB-I-GPE\n",
      "TX-B-GPE\n",
      "78236-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "MISC-O-\n",
      "GENERAL-O-\n",
      "MERCHANDISE-O-\n",
      "-------------------------------------------------------\n",
      "AAFES-O-\n",
      "PINES-O-\n",
      "SHOPPETTE-O-\n",
      "FORT-O-\n",
      "BRAGG-O-\n",
      "XX-O-\n",
      "283070000-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "MISC-O-\n",
      "GENERAL-O-\n",
      "MERCHANDISE-O-\n",
      "-------------------------------------------------------\n",
      "ABC-O-\n",
      "MS-O-\n",
      "GILBERT-O-\n",
      "AZ-B-GPE\n",
      "85297-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "MEMBER-O-\n",
      "CLUBS-O-\n",
      "/-O-\n",
      "SPORT-O-\n",
      "/-O-\n",
      "REC-O-\n",
      "/-O-\n",
      "GOLF-O-\n",
      "-------------------------------------------------------\n",
      "ACTIVA-O-\n",
      "LEDYARD-O-\n",
      "CT-O-\n",
      "6338-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "MISC-O-\n",
      "APPAREL-O-\n",
      "/-O-\n",
      "ACCESS-O-\n",
      "STORES-O-\n",
      "-------------------------------------------------------\n",
      "ANGELA-B-ORG\n",
      "FOOD-I-ORG\n",
      "LLC-I-ORG\n",
      "COLUMBUS-B-GPE\n",
      "OH-B-GPE\n",
      "43207-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "GROCERY-O-\n",
      "STORES-O-\n",
      "/-O-\n",
      "SUPERMARKETS-O-\n",
      "-------------------------------------------------------\n",
      "ANIME-O-\n",
      "AGES-O-\n",
      "RCH-O-\n",
      "CUCAMONGA-B-GPE\n",
      "CA-B-GPE\n",
      "91730-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "DVD-O-\n",
      "/-O-\n",
      "VIDEO-O-\n",
      "TAPE-O-\n",
      "RENTAL-O-\n",
      "STORES-O-\n",
      "-------------------------------------------------------\n",
      "BHC-O-\n",
      "MERRILL-O-\n",
      "RANCH-O-\n",
      "LUKEVILLE-B-GPE\n",
      "AZ-B-GPE\n",
      "85341-B-CARDINAL\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "HOSPITALS-O-\n",
      "-------------------------------------------------------\n",
      "CHRISTOPHER-B-PERSON\n",
      "KOCH-I-PERSON\n",
      "MATCO-I-PERSON\n",
      "CHANDLER-B-GPE\n",
      "AZ-B-GPE\n",
      "85286-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "HARDWARE-O-\n",
      "EQUIPMENT-O-\n",
      "/-O-\n",
      "SUPPLIES-O-\n",
      "-------------------------------------------------------\n",
      "CUMING-O-\n",
      "CO-O-\n",
      "MOTOR-O-\n",
      "VEH-O-\n",
      "CARROLLTON-O-\n",
      "TX-B-GPE\n",
      "75006-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "GOV'T-O-\n",
      "SERV-O-\n",
      "--O-\n",
      "DEFAULT-O-\n",
      "-------------------------------------------------------\n",
      "DJANDJ-B-ORG\n",
      "INVESTMENTS-I-ORG\n",
      "INC-I-ORG\n",
      "SAN-B-GPE\n",
      "ANTONIO-I-GPE\n",
      "TX-B-GPE\n",
      "78238-B-CARDINAL\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "INSURANCE-O-\n",
      "SALES-O-\n",
      "/-O-\n",
      "UNDERWRITE-O-\n",
      "-------------------------------------------------------\n",
      "GCA-O-\n",
      "EMMETSBURG-O-\n",
      "IA-O-\n",
      "50536-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "WIRE-O-\n",
      "TRANSFER-O-\n",
      "MONEY-O-\n",
      "ORDER-O-\n",
      "-------------------------------------------------------\n",
      "GOOGNFC-O-\n",
      "HAGGEN-O-\n",
      "02189-O-\n",
      "MOUNTAIN-B-GPE\n",
      "VIEW-I-GPE\n",
      "CA-O-\n",
      "94043-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "GROCERY-O-\n",
      "STORES-O-\n",
      "/-O-\n",
      "SUPERMARKETS-O-\n",
      "-------------------------------------------------------\n",
      "LCWM-O-\n",
      "VIDA-O-\n",
      "LEABURG-O-\n",
      "OR-O-\n",
      "97489-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "GOV'T-O-\n",
      "SERV-O-\n",
      "--O-\n",
      "DEFAULT-O-\n",
      "-------------------------------------------------------\n",
      "MELTON-O-\n",
      "CONCESSIONS-O-\n",
      "MILLTOWN-O-\n",
      "IN-O-\n",
      "47145-B-CARDINAL\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "OPAY-O-\n",
      "ADA-O-\n",
      "SHERIFF-O-\n",
      "MISD-O-\n",
      "NORCROSS-O-\n",
      "GA-O-\n",
      "30092-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "GOV'T-O-\n",
      "SERV-O-\n",
      "--O-\n",
      "DEFAULT-O-\n",
      "-------------------------------------------------------\n",
      "SEACRET-O-\n",
      "1-O-\n",
      "ENGLEWOOD-O-\n",
      "CO-O-\n",
      "80112-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "COSMETIC-O-\n",
      "STORES-O-\n",
      "-------------------------------------------------------\n",
      "SEARS-O-\n",
      "ROEBUCK-O-\n",
      "NEPTUNE-B-GPE\n",
      "NJ-B-GPE\n",
      "7753-B-CARDINAL\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "DEPARTMENT-O-\n",
      "STORES-O-\n",
      "-------------------------------------------------------\n",
      "SILOAM-O-\n",
      "SMOKE-O-\n",
      "SHOP-O-\n",
      "WATTS-O-\n",
      "OK-O-\n",
      "74964-B-CARDINAL\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "CIGAR-O-\n",
      "STORES-O-\n",
      "/-O-\n",
      "STANDS-O-\n",
      "-------------------------------------------------------\n",
      "SLEEP-O-\n",
      "TRAIN-O-\n",
      "00328-O-\n",
      "CRESTLINE-O-\n",
      "CA-O-\n",
      "92325-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "FURNITURE-O-\n",
      "/-O-\n",
      "EQUIP-O-\n",
      "STORES-O-\n",
      "-------------------------------------------------------\n",
      "SLEEP-O-\n",
      "TRAIN-O-\n",
      "00637-O-\n",
      "PORTLAND-B-GPE\n",
      "OR-O-\n",
      "97211-B-CARDINAL\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "FURNITURE-O-\n",
      "/-O-\n",
      "EQUIP-O-\n",
      "STORES-O-\n",
      "-------------------------------------------------------\n",
      "UHI-O-\n",
      "JUPITER-O-\n",
      "FL-B-GPE\n",
      "33458-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "TRUCK-O-\n",
      "/-O-\n",
      "UTILITY-O-\n",
      "TRAILER-O-\n",
      "RENTALS-O-\n",
      "-------------------------------------------------------\n",
      "USAARAMARK-B-ORG\n",
      "REFRESHMENTS-I-ORG\n",
      "NORCROSS-O-\n",
      "GA-B-GPE\n",
      "30092-B-CARDINAL\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "FAST-O-\n",
      "FOOD-O-\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "USACANTEEN-O-\n",
      "VENDING-O-\n",
      "PALM-B-GPE\n",
      "HARBOR-I-GPE\n",
      "FL-I-GPE\n",
      "34683-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "FAST-O-\n",
      "FOOD-O-\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "USAHURRICANE-O-\n",
      "SIMULATOR-O-\n",
      "CORONA-O-\n",
      "CA-O-\n",
      "92879-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "FAST-O-\n",
      "FOOD-O-\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "USAVEND-O-\n",
      "AT-O-\n",
      "AIR-O-\n",
      "SERV-O-\n",
      "TUCSON-B-GPE\n",
      "AZ-B-GPE\n",
      "85755-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "MISC-O-\n",
      "SPECIALTY-O-\n",
      "RETAIL-O-\n",
      "-------------------------------------------------------\n",
      "USAVEND-O-\n",
      "AT-O-\n",
      "AIR-O-\n",
      "SERV-O-\n",
      "SUGAR-B-GPE\n",
      "HILL-I-GPE\n",
      "GA-I-GPE\n",
      "30518-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "MISC-O-\n",
      "SPECIALTY-O-\n",
      "RETAIL-O-\n",
      "-------------------------------------------------------\n",
      "USAVEND-O-\n",
      "AT-O-\n",
      "COCA-B-FAC\n",
      "--I-FAC\n",
      "COLA-I-FAC\n",
      "DALLAS-I-FAC\n",
      "NC-O-\n",
      "28034-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "FAST-O-\n",
      "FOOD-O-\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "VENDING-O-\n",
      "BRANDON-O-\n",
      "ROBB-O-\n",
      "DES-O-\n",
      "MOINES-O-\n",
      "IA-O-\n",
      "50310-B-CARDINAL\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "FAST-O-\n",
      "FOOD-O-\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "VENDING-O-\n",
      "CT-O-\n",
      "VENDING-O-\n",
      "CASTLE-O-\n",
      "PINES-O-\n",
      "CO-O-\n",
      "80108-B-CARDINAL\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "FAST-O-\n",
      "FOOD-O-\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n",
      "VENDING-O-\n",
      "EDWARD-O-\n",
      "MCINNIS-O-\n",
      "MI-B-GPE\n",
      "FREELAND-B-GPE\n",
      "MI-B-GPE\n",
      "48623-O-\n",
      "UNITED-B-GPE\n",
      "STATES-I-GPE\n",
      "OF-I-GPE\n",
      "AMERICA-I-GPE\n",
      "FAST-O-\n",
      "FOOD-O-\n",
      "RESTAURANTS-O-\n",
      "-------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "for index, row in test_data.iterrows():\n",
    "    doc = nlp(row['Init'])\n",
    "    for token in doc:\n",
    "        print(token.text + \"-\" + token.ent_iob_ + \"-\" + token.ent_type_)\n",
    "    print(\"-------------------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://google.com/search?q=LOWES+02945++KY++UNITED+STATES+OF+AMERICA+HOME+SUPPLY+WAREHOUSE+STORES\n",
      "Response HTTP Status Code:  200\n"
     ]
    }
   ],
   "source": [
    "links = []\n",
    "titles = []\n",
    "snippets = []\n",
    "i = 0\n",
    "\n",
    "text1= '14 RBRT GAHANNA COLUMBUS OH 43230 RESTAURANTS'\n",
    "text2= 'AAFES GUAM NEW MAIN STOR DALLAS TX 75265 UNITED STATES OF AMERICA MISC GENERAL MERCHANDISE'\n",
    "text3= 'GAMESTOP.COM 19 2700 MIA DAYTON OH 45459 UNITED STATES OF AMERICA VIDEO AMUSEMENT GAME SUPPLY'\n",
    "text4= 'SP  CHARGEDD HTTPSCHARGEDE MO 65201 UNITED STATES OF AMERICA ELECTRONICS STORES'\n",
    "text5= 'UBER  CA  UNITED STATES OF AMERICA TAXICABS/LIMOUSINES'\n",
    "text6= 'FRISCO LAKES GOLF CLUB WESTLAKEA TX 76262 UNITED STATES OF AMERICA PUBLIC GOLF COURSES'\n",
    "text7 = 'HUA FU FAT BULL SEAFOO RIVERSIDE CA 92507 UNITED STATES OF AMERICA RESTAURANTS'\n",
    "text8 = '860 LSU TEAM SHOP GREENWICH CT 6830 UNITED STATES OF AMERICA COSMETIC STORES'\n",
    "textx = 'JARED GALLERIA 12430 FOOT  CA  UNITED STATES OF AMERICA JEWELRY STORES'\n",
    "text9 = 'LOWES 02945  KY  UNITED STATES OF AMERICA HOME SUPPLY WAREHOUSE STORES'\n",
    "\n",
    "search_query_list = [textx]\n",
    "init_search_query = search_query_list[0]\n",
    "\n",
    "search_query_proc_docs = list(textPreproc(nlp, search_query_list))\n",
    "\n",
    "\n",
    "base_url = 'https://google.com/search?'\n",
    "base_url_dym = 'https://google.com'\n",
    "txt = {'q': init_search_query}\n",
    "search_url = base_url + urllib.parse.urlencode(txt)\n",
    "# search_url = 'http://httpbin.org/anything?json'\n",
    "print(search_url)\n",
    "params = {'url': search_url, 'custom_google':True}\n",
    "request_result = send_request(params)\n",
    "\n",
    "# parsing the available content\n",
    "soup = BeautifulSoup(request_result.content, 'lxml')\n",
    "did_you_mean = soup.find('a', {'class': 'gL9Hy'})\n",
    "did_you_mean_link = \"\"\n",
    "while did_you_mean is not None:\n",
    "    did_you_mean_link = did_you_mean.get('href')\n",
    "    did_you_mean_link = base_url_dym + did_you_mean_link\n",
    "    print(did_you_mean_link)\n",
    "    params = {'url': did_you_mean_link, 'custom_google':True}\n",
    "    request_result = send_request(params)\n",
    "    soup = BeautifulSoup(request_result.content, 'lxml')\n",
    "    did_you_mean = soup.find('a', {'class': 'gL9Hy'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(soup.prettify())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No local listings\n"
     ]
    }
   ],
   "source": [
    "local_listings = soup.find('div', {'class': 'UDZeY OTFaAf'})\n",
    "if local_listings is not None:\n",
    "    tree = etree.HTML(str(local_listings))\n",
    "    format_tags = ['b', 'strong', 'i', 'em', 'mark', 'small', 'del', 'ins', 'sub', 'sup']\n",
    "    # root = tree.getroot()\n",
    "\n",
    "    text_list = []\n",
    "    for element in tree.iter():\n",
    "        text_to_append = ''\n",
    "        if element.tag not in ['script', 'style', 'head', 'div'] and element.text is not None:\n",
    "            text_to_append = element.text\n",
    "            #print(\"In tag: \" + element.tag, end=\"\")\n",
    "            #print(\"--- new text: \" + text_to_append)\n",
    "        \n",
    "        if element.tag in format_tags and len(text_list) > 0:\n",
    "            text_to_append = text_list[-1] + text_to_append\n",
    "            text_list[-1] = text_to_append\n",
    "            #print(\"In tag: \" + element.tag, end=\"\")\n",
    "            #print(\"--- new text: \" + text_to_append)\n",
    "        elif text_to_append != '':\n",
    "            text_list.append(text_to_append)\n",
    "\n",
    "    print(text_list)\n",
    "    text_list_lr = list(map(lambda x: x.lower(), text_list))\n",
    "    if 'address' in text_list_lr and 'address' != text_list_lr[-1]:\n",
    "        req_addr_ind = text_list_lr.index('address')\n",
    "        req_addr = text_list[req_addr_ind + 1]\n",
    "        print(\"\\nExtracted address: \", end=\"\")\n",
    "        print(req_addr)\n",
    "else:\n",
    "    print(\"No local listings\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No local pack\n"
     ]
    }
   ],
   "source": [
    "local_listings_full = soup.find('div', {'class': 'ifM9O'})\n",
    "local_listings = soup.find('div', {'class': 'UDZeY OTFaAf'})\n",
    "\n",
    "local_pk = soup.find_all('div', {'class': 'cXedhc'})\n",
    "if len(local_pk) != 0:\n",
    "    for item in local_pk:\n",
    "        name_elt = item.find('div', {'class': 'dbg0pd'})\n",
    "        if name_elt is not None:\n",
    "            name = name_elt.find('span').getText()\n",
    "        addr_elt = item.find('div', {'class':'rllt__details'})\n",
    "        if addr_elt is not None:\n",
    "            addrs = addr_elt.find_all('div')\n",
    "            addr_strs = []\n",
    "            for addr in addrs:\n",
    "                addr_strs.append(addr.getText())\n",
    "        print(\"Name: \" + name)\n",
    "        for addr_str in addr_strs:\n",
    "            print(\"Address: \" + addr_str)\n",
    "        print(\"--------------------------\")\n",
    "else:\n",
    "    print(\"No local pack\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.jared.com/\n",
      "Jared | Your Local Jewelry Store for Engagement Rings ...\n",
      "Jared is America's home for fine jewelry including diamond jewelry, gold jewelry, and engagement rings. We also carry a great selection of color gemstone ...\n",
      "\n",
      "\n",
      "https://m.yelp.com/biz/jared-rancho-cucamonga\n",
      "Jared - 12430 Foothill Blvd, Rancho Cucamonga, CA - Yelp\n",
      "98 reviews of Jared \"Every time we've been in here, the staff is really helpful. ... If you're in the market for jewelry, Jared is at least worth a look.\n",
      "\n",
      "\n",
      "https://www.businessyab.com/explore/united_states/california/san_bernardino_county/rancho_cucamonga/east_foothill_boulevard/12430/jared.html\n",
      "Jared, 12430 E Foothill Blvd, Rancho Cucamonga, CA 91739 ...\n",
      "12430 E Foothill Blvd, Rancho Cucamonga, CA 91739, USA. Jared is a business providing services in the field of Jewelry store, Store, .\n",
      "\n",
      "\n",
      "https://www.mapquest.com/us/california/jared-42288108\n",
      "Jared 12430 Foothill Blvd Rancho Cucamonga, CA Jewelers\n",
      "Get directions, reviews and information for Jared in Rancho Cucamonga, CA. ... American Express; Discover; MasterCard; Store Card; Visa ... Let us know!\n",
      "\n",
      "\n",
      "https://www.loc8nearme.com/california/rancho-cucamonga/jared-the-galleria-of-jewelry/1130556/\n",
      "Jared Rancho Cucamonga, CA 91739 - 12430 Foothill Blvd\n",
      "We offer five times the selection of ordinary jewelry stores, price it well, and present it with the help of a team of experts, thereby creating the ultimate ...\n",
      "\n",
      "\n",
      "https://foursquare.com/v/jared/4c423bacd7fad13ac6da08da\n",
      "Jared - Jewelry Store in Rancho Cucamonga - Foursquare\n",
      "See 4 photos and 1 tip from 102 visitors to Jared. \"Love this place. Great service from Janelle and Michelle with my Pandora.\"\n",
      "\n",
      "\n",
      "https://vymaps.com/US/Jared-The-Galleria-of-Jewelry-2196961/\n",
      "Jared The Galleria of Jewelry, California (+1 909-463-6161)\n",
      "12430 E Foothill Blvd, Rancho Cucamonga, CA 91739, USA | Jewelry Store, Store.\n",
      "\n",
      "\n",
      "https://www.starcojewellers.com.au/j-jewelry/jareds-jewelry-rancho-cucamonga.html\n",
      "Jareds Jewelry Rancho Cucamonga - Starco Jewellers\n",
      "Jared Jewelry Rancho Cucamonga CA - Victoria Gardens 12430 Foothill Store Location ... Gardens 12430 Foothill Blvd. Rancho Cucamonga, CA 91730 – USA - Store ...\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "search = soup.find_all('div', {'class': 'tF2Cxc'})\n",
    "#rso > div:nth-child(9) > div > div > div.IsZvec > div\n",
    "for h in search:\n",
    "    links.append(h.a.get('href'))\n",
    "    titles.append(h.a.find('h3', {'class':'LC20lb DKV0Md'}).getText())\n",
    "    print(links[i])\n",
    "    print(titles[i])\n",
    "    spans1 = h.find('div', {'class':'IsZvec'})\n",
    "    if spans1 is not None:\n",
    "        spans2 = spans1.find('div', {'class': 'VwiC3b yXK7lf MUxGbd yDYNvb lyLwlc lEBKkf'})\n",
    "        if spans2 is not None:\n",
    "            spans = spans2.find_all('span')\n",
    "            if len(spans) > 0:\n",
    "                snippets.append(spans[-1].getText())\n",
    "            else:\n",
    "                snippets.append(\"\")\n",
    "        else:\n",
    "            snippets.append(\"\")\n",
    "    print(snippets[i])\n",
    "    \n",
    "    #spans = h.find('div.IsZvec').find('div.VwiC3b.yXK7lf.MUxGbd.yDYNvb.lyLwlc.lEBKkf').find_all('span')\n",
    "    #snippets.append(spans[-1].getText())\n",
    "    #print(links[i])\n",
    "    #print(titles[i])\n",
    "    #print(snippets[i])\n",
    "    print(\"\\n\")\n",
    "    i += 1\n",
    "p = input(\"Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "((\\d+[:]CARDINAL[;])*(\\d+[:]FAC[;])*(\\d+[:]PUNCT[;])*(\\d+[:]LOC[;])*(\\d+[:]PUNCT[;])*(\\d+[:]ORDINAL[;])*(\\d+[:]PUNCT[;])*((\\d+[:]FAC[;])*(\\d+[:]PUNCT[;])*)*(\\d+[:]GPE[;])+(\\d+[:]PUNCT[;])*((\\d+[:]GPE[;])*(\\d+[:]PUNCT[;])*)*(\\d+[:]CARDINAL[;])*(\\d+[:]PUNCT[;])*(\\d+[:]GPE[;])*)|(((\\d+[:]PRODUCT[;])|(\\d+[:]WORK_OF_ART[;]))*(\\d+[:]ORG[;])+)\n",
      "he\\lloth\\ere\n"
     ]
    }
   ],
   "source": [
    "print(par_open + addr_pattern + par_close + or_pipe + par_open + biz_name_pattern + par_close)\n",
    "string1 = 'he\\lloth\\ere'\n",
    "print(string1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['0:NOENT;1:NOENT;2:NOENT;3:NOENT;4:NOENT;5:NOENT;6:NOENT;7:ORG;8:NOENT;9:NOENT;10:NOENT;11:NOENT;12:NOENT;13:ORG;14:NOENT;15:PUNCT;16:NOENT;17:PUNCT;18:NOENT;19:NOENT;20:NOENT;21:PUNCT;22:NOENT;23:ORG;24:NOENT;25:NOENT;26:NOENT;27:NOENT;28:NOENT;', '0:NOENT;1:NOENT;2:PUNCT;3:NOENT;4:NOENT;5:GPE;6:GPE;7:PUNCT;8:GPE;9:PUNCT;10:ORG;11:ORG;12:ORG;13:NOENT;14:NOENT;15:NOENT;16:NOENT;17:NOENT;18:NOENT;19:NOENT;20:NOENT;21:NOENT;22:NOENT;23:GPE;24:NOENT;25:NOENT;26:NOENT;27:NOENT;28:NOENT;29:NOENT;', '0:FAC;1:FAC;2:FAC;3:PUNCT;4:NOENT;5:NOENT;6:NOENT;7:NOENT;8:NOENT;9:NOENT;10:NOENT;11:NOENT;12:ORG;13:ORG;14:PUNCT;15:NOENT;16:NOENT;17:DATE;18:DATE;19:DATE;20:DATE;21:PUNCT;22:NOENT;23:NOENT;24:NOENT;25:EVENT;26:EVENT;27:NOENT;', '0:ORG;1:ORG;2:ORG;3:ORG;4:ORG;5:NOENT;6:NOENT;7:NOENT;8:NOENT;9:ORG;10:NOENT;11:NOENT;12:NOENT;13:NOENT;14:PUNCT;15:NOENT;16:NOENT;17:ORG;18:ORG;19:NOENT;20:NOENT;21:PUNCT;22:ORG;23:NOENT;24:NOENT;25:PUNCT;26:ORG;27:NOENT;28:NOENT;29:NOENT;30:NOENT;31:NOENT;32:NOENT;', '0:ORG;1:ORG;2:ORG;3:ORG;4:ORG;5:ORG;6:NOENT;7:NOENT;8:CARDINAL;9:PUNCT;10:GPE;11:PUNCT;12:GPE;13:GPE;14:PUNCT;15:CARDINAL;16:PUNCT;17:NOENT;18:CARDINAL;19:PUNCT;20:ORG;21:ORG;22:ORG;23:ORG;24:ORG;25:ORG;26:ORG;', '0:NOENT;1:NOENT;2:NOENT;3:NOENT;4:NOENT;5:PUNCT;6:PUNCT;7:NOENT;8:NOENT;9:NOENT;10:ORG;11:ORG;12:ORG;13:ORG;14:NOENT;15:NOENT;16:NOENT;17:NOENT;18:NOENT;19:NOENT;20:PUNCT;21:NOENT;22:PUNCT;23:NOENT;24:PUNCT;25:NOENT;26:NOENT;27:NOENT;28:NOENT;29:NOENT;30:NOENT;', '0:ORG;1:ORG;2:ORG;3:ORG;4:NOENT;5:NOENT;6:NOENT;7:NOENT;8:NOENT;9:NOENT;10:NOENT;11:NOENT;12:PUNCT;13:NOENT;14:PUNCT;15:NOENT;16:NOENT;17:NOENT;18:CARDINAL;19:NOENT;20:NOENT;21:CARDINAL;22:CARDINAL;23:CARDINAL;24:NOENT;25:NOENT;', '0:CARDINAL;1:NOENT;2:NOENT;3:NOENT;4:NOENT;5:NOENT;6:NOENT;7:NOENT;8:NOENT;9:NOENT;10:NOENT;11:NOENT;12:NOENT;13:NOENT;14:NOENT;15:ORG;16:NOENT;17:ORG;', '0:NOENT;1:NOENT;2:ORG;3:ORG;4:ORG;5:ORG;6:PUNCT;7:ORG;8:ORG;9:ORG;10:ORG;11:ORG;12:PUNCT;13:ORG;14:PUNCT;15:PUNCT;16:NOENT;17:ORG;18:ORG;19:ORG;20:ORG;21:ORG;22:ORG;23:ORG;24:NOENT;25:NOENT;26:NOENT;27:FAC;28:NOENT;29:FAC;30:NOENT;', '0:NOENT;1:ORG;2:NOENT;3:NOENT;4:NOENT;5:GPE;6:NOENT;7:NOENT;8:NOENT;9:NOENT;10:NOENT;11:NOENT;12:NOENT;13:NOENT;14:NOENT;15:NOENT;16:NOENT;17:NOENT;18:NOENT;19:NOENT;20:NOENT;21:PUNCT;22:NOENT;23:NOENT;']\n",
      "['0:ORG;1:NOENT;2:NOENT;3:PUNCT;4:ORG;5:NOENT;6:NOENT;7:PUNCT;8:ORG;9:ORG;10:NOENT;', '0:NOENT;1:NOENT;2:CARDINAL;3:NOENT;4:PUNCT;5:NOENT;6:NOENT;7:NOENT;8:GPE;9:PUNCT;10:GPE;', '0:ORG;1:NOENT;2:PUNCT;3:NOENT;4:PUNCT;5:NOENT;6:NOENT;', '0:NOENT;1:NOENT;2:NOENT;3:NOENT;4:PUNCT;5:NOENT;6:NOENT;7:ORG;8:NOENT;9:NOENT;', '0:NOENT;1:PUNCT;2:GPE;3:GPE;4:ORG;5:ORG;6:ORG;', '0:NOENT;1:NOENT;2:PUNCT;3:GPE;4:PUNCT;5:GPE;6:PUNCT;7:ORG;8:ORG;9:ORG;10:ORG;', '0:ORG;1:ORG;2:ORG;3:PUNCT;4:ORG;5:ORG;6:ORG;7:NOENT;', '0:ORG;1:NOENT;2:PUNCT;3:NOENT;4:NOENT;5:NOENT;6:GPE;7:NOENT;8:NOENT;9:PUNCT;10:ORG;11:ORG;', '0:NOENT;1:NOENT;2:NOENT;3:LOC;4:LOC;5:DATE;6:DATE;7:DATE;8:NOENT;9:PUNCT;10:NOENT;', '0:ORG;1:NOENT;2:NOENT;3:NOENT;4:PUNCT;5:ORG;']\n"
     ]
    }
   ],
   "source": [
    "in_strings = [\n",
    "    \"AAFES KEESLER SHE,, MSCSS ,HATTIESBURG MS 39407 UNITED STATES OF AMERICA MISC GENERAL MERCHANDISE\"\n",
    ", \"Bldg..... 2600 Camp-34567-Shelby,,, MS 39407----2003 United States\"\n",
    ", \"AAFES KEESLER SHE,, MSCSS ,HATTIESBURG MS 39407 UNITED STATES OF AMERICA MISC GENERAL MERCHANDISE\"\n",
    ", \"Building 2600 Camp Shelby, MS 39407-2003 United States\"\n",
    ", \"GAMESTOP.COM 19 2700 MIA DAYTON OH 45459 UNITED STATES OF AMERICA VIDEO AMUSEMENT GAME SUPPLY\"\n",
    "]\n",
    "out_snippets = textPreproc(nlp, snippets)\n",
    "out_titles = textPreproc(nlp, titles)\n",
    "orig_snippets = list(out_snippets)\n",
    "orig_titles = list(out_titles)\n",
    "# for i in range(len(orig_strings)):\n",
    "#     print(orig_strings[i])\n",
    "#     print(\"Type: \" + str(type(orig_strings[i])))\n",
    "# stringInfo(out_strings)\n",
    "ent_snippets = entityString(orig_snippets)\n",
    "ent_titles = entityString(orig_titles)\n",
    "print(ent_snippets)\n",
    "print(ent_titles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "search_results_addr = extractPattern(addr_pattern, ent_snippets, ADDRESS)\n",
    "search_results_biz = extractPattern(biz_name_pattern, ent_titles, BIZ_NAME)\n",
    "matches_across_search_results_addr = getMatches(search_results_addr, orig_snippets)\n",
    "matches_across_search_results_biz = getMatches(search_results_biz, orig_titles)\n",
    "#match_list = [matches_across_search_results_addr, matches_across_search_results_biz]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "860 LSU TEAM SHOP GREENWICH Connecticut 6830  COSMETIC STORES\n",
      "\n",
      "Biz names found: LSU, LSU, LSU Tigers, \n",
      "No matches found in result 1\n",
      "------------------------------------------------------\n",
      "\n",
      "Biz names found: None\n",
      "\n",
      "No matches found in result 2\n",
      "------------------------------------------------------\n",
      "\n",
      "Biz names found: LSU, \n",
      "No matches found in result 3\n",
      "------------------------------------------------------\n",
      "\n",
      "Biz names found: LSU, \n",
      "No matches found in result 4\n",
      "------------------------------------------------------\n",
      "\n",
      "Biz names found: Department of Justice, 138. Franconia. New Hampshire. 03580\n",
      "The Fuzz Score (partial ratio): 22\n",
      "The Fuzz Score (token set ratio): 39\n",
      "The Fuzz Score (token sort ratio): 39\n",
      "Link: https://www.doj.nh.gov/charitable-trusts/documents/registered-charities.pdf\n",
      "------------------------------------------------------\n",
      "\n",
      "Biz names found: The Cosmetics Company Store, \n",
      "No matches found in result 6\n",
      "------------------------------------------------------\n",
      "\n",
      "Biz names found: LSU Alumni Association, Louisiana State University, \n",
      "No matches found in result 7\n",
      "------------------------------------------------------\n",
      "\n",
      "Biz names found: LSU, LSU Wire, \n",
      "No matches found in result 8\n",
      "------------------------------------------------------\n",
      "\n",
      "Biz names found: None\n",
      "\n",
      "No matches found in result 9\n",
      "------------------------------------------------------\n",
      "\n",
      "Biz names found: LSU, Walmart, \n",
      "No matches found in result 10\n",
      "------------------------------------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "search_query_proc = search_query_proc_docs[0].text\n",
    "print(search_query_proc, end=\"\\n\\n\")\n",
    "#print(matches_across_search_results)\n",
    "for (k, v) in matches_across_search_results_addr.items():\n",
    "    print(\"Biz names found: \", end=\"\")\n",
    "    if len(matches_across_search_results_biz[k]) > 0:\n",
    "        for match in matches_across_search_results_biz[k]:\n",
    "            print(match, end=\", \")\n",
    "    else:\n",
    "        print(\"None\")    \n",
    "    if len(v) > 0:\n",
    "        for match in v:\n",
    "            print(match)\n",
    "            print(\"The Fuzz Score (partial ratio): \" + str(fuzz.partial_ratio(search_query_proc, match)))\n",
    "            print(\"The Fuzz Score (token set ratio): \" + str(fuzz.token_set_ratio(search_query_proc, match)))\n",
    "            print(\"The Fuzz Score (token sort ratio): \" + str(fuzz.token_sort_ratio(search_query_proc, match)))\n",
    "        print(\"Link: \" + links[k])\n",
    "    else:\n",
    "        print(\"\\nNo matches found in result \" + str(k + 1))\n",
    "    print(\"------------------------------------------------------\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('a', 'b', '', 'c', '', 'b')\n"
     ]
    }
   ],
   "source": [
    "pattern = \"(a)*(b)+(a)*(c)+(a)*(b)*\"\n",
    "string_eg = \"ttttttahahggfdgfgfaabbbbcccbbbbbbtfufyfu\"\n",
    "matches = re.findall(pattern, string_eg)\n",
    "for match in matches:\n",
    "    print(match)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GAMESTOP.COM-O-\n",
      "19-O-\n",
      "2700-O-\n",
      "MIA-O-\n",
      "DAYTON-O-\n",
      "OH-B-GPE\n",
      "4545-O-\n",
      "VIDEO-O-\n",
      "AMUSEMENT-O-\n",
      "GAME-O-\n",
      "SUPPLY-O-\n"
     ]
    }
   ],
   "source": [
    "doc = nlp(\"GAMESTOP.COM 19 2700 MIA DAYTON OH 4545 VIDEO AMUSEMENT GAME SUPPLY\")\n",
    "for token in doc:\n",
    "    print(token.text + \"-\" + token.ent_iob_ + \"-\" + token.ent_type_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "91\n",
      "100\n",
      "91\n",
      "91\n"
     ]
    }
   ],
   "source": [
    "string1 = \"a b c\"\n",
    "string2 = \"b c d\"\n",
    "string3 = \"a b cd\"\n",
    "print(str(fuzz.ratio(string1, string3)))\n",
    "print(str(fuzz.partial_ratio(string1, string3)))\n",
    "print(str(fuzz.token_set_ratio(string1, string3)))\n",
    "print(str(fuzz.token_sort_ratio(string1, string3)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Levenshtein distance function"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "            __\n",
    "            |   |a|    ------------------------------------------------------   if |b| = 0,\n",
    "            |   |b|    ------------------------------------------------------   if |a| = 0,\n",
    "lev(a, b) = <   lev(tail(a), tail(b))  ---------------------------------------  if a[0] = b[0],\n",
    "            |           __\n",
    "            |           |   lev(tail(a), b) [deletion, from string 1 to 2]\n",
    "            |   1 + min <   lev(a, tail(b)) [insertion, from string 1 to 2]     otherwise.\n",
    "            |_          |_  lev(tail(a), tail(b)) [substitution in string 1]\n",
    "\n",
    "tail(x) = x[1: n], n = len(x)\n",
    "                |\n",
    "                --> last index not included"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Example\n",
    "For example, the Levenshtein distance between \"kitten\" and \"sitting\" is 3, since the following three edits change one into the other, and there is no way to do it with fewer than three edits:\n",
    "\n",
    "- kitten → sitten (substitution of \"s\" for \"k\")\n",
    "- sitten → sittin (substitution of \"i\" for \"e\")\n",
    "- sittin → sitting (insertion of \"g\" at the end)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Relationship with other edit distance metrics\n",
    "There are other popular measures of edit distance, which are calculated using a different set of allowable edit operations. For instance,\n",
    "\n",
    "- the Damerau–Levenshtein distance allows the transposition of two adjacent characters alongside insertion, deletion, substitution;\n",
    "- the longest common subsequence (LCS) distance allows only insertion and deletion, not substitution;\n",
    "- the Hamming distance allows only substitution, hence, it only applies to strings of the same length.\n",
    "- the Jaro distance allows only transposition.\n",
    "\n",
    "Edit distance is usually defined as a parameterizable metric calculated with a specific set of allowed edit operations, and each operation is assigned a cost (possibly infinite). This is further generalized by DNA sequence alignment algorithms such as the Smith–Waterman algorithm, which make an operation's cost depend on where it is applied."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Upper and lower bounds\n",
    "The Levenshtein distance has several simple upper and lower bounds. These include:\n",
    "\n",
    "- It is at least the difference of the sizes of the two strings.\n",
    "- It is at most the length of the longer string.\n",
    "- It is zero if and only if the strings are equal.\n",
    "- If the strings are the same size, the Hamming distance is an upper bound on the Levenshtein distance. The Hamming distance is the number of positions at which the corresponding symbols in the two strings are different.\n",
    "- The Levenshtein distance between two strings is no greater than the sum of their Levenshtein distances from a third string (triangle inequality).\n",
    "\n",
    "An example where the Levenshtein distance between two strings of the same length is strictly less than the Hamming distance is given by the pair \"flaw\" and \"lawn\". Here the Levenshtein distance equals 2 (delete \"f\" from the front; insert \"n\" at the end). The Hamming distance is 4."
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "118d9abf58a59593377585876c470c3878d343a2e69bdda12d374a5576ff019d"
  },
  "kernelspec": {
   "display_name": "Python 3.9.6 64-bit (conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}