Dog_name_scrapping,dog_name_cleaning_and_dog_info_scrapping.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U7ts9CHupa5f",
        "outputId": "e5f477a2-2adc-47c9-e5b2-c6f38f8255b1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['Affenpinscher', 'Afghan-Hound', 'Airedale-Terrier', 'Akita', 'Alaskan-Klee-Kai', 'Alaskan-Malamute', 'American-Bulldog', 'American-English-Coonhound', 'American-Eskimo-Dog', 'American-Foxhound', 'American-Hairless-Terrier', 'American-Leopard-Hound', 'American-Staffordshire-Terrier', 'American-Water-Spaniel', 'Anatolian-Shepherd-Dog', 'Appenzeller-Sennenhund', 'Australian-Cattle-Dog', 'Australian-Kelpie', 'Australian-Shepherd', 'Australian-Stumpy-Tail-Cattle-Dog', 'Australian-Terrier', 'Azawakh', 'Barbado-da-Terceira', 'Barbet', 'Basenji', 'Basset-Fauve-de-Bretagne', 'Basset-Hound', 'Bavarian-Mountain-Scent-Hound', 'Beagle', 'Bearded-Collie', 'Beauceron', 'Bedlington-Terrier', 'Belgian-Laekenois', 'Belgian-Malinois', 'Belgian-Sheepdog', 'Belgian-Tervuren', 'Bergamasco-Sheepdog', 'Berger-Picard', 'Bernese-Mountain-Dog', 'Bichon-Frise', 'Biewer-Terrier', 'Black-and-Tan-Coonhound', 'Black-Russian-Terrier', 'Bloodhound', 'Bluetick-Coonhound', 'Boerboel', 'Bohemian-Shepherd', 'Bolognese', 'Border-Collie', 'Border-Terrier', 'Borzoi', 'Boston-Terrier', 'Bouvier-des-Flandres', 'Boxer', 'Boykin-Spaniel', 'Bracco-Italiano', 'Braque-du-Bourbonnais', 'Braque-Francais-Pyrenean', 'Braque-Saint-Germain', 'Brazilian-Terrier', 'Briard', 'Brittany', 'Broholmer', 'Brussels-Griffon', 'Bull-Terrier', 'Bulldog', 'Bullmastiff', 'Cairn-Terrier', 'Canaan-Dog', 'Cane-Corso', 'Cardigan-Welsh-Corgi', 'Carolina-Dog', 'Catahoula-Leopard-Dog', 'Caucasian-Shepherd-Dog', 'Cavalier-King-Charles-Spaniel', 'Central-Asian-Shepherd-Dog', 'Cesky-Terrier', 'Chesapeake-Bay-Retriever', 'Chihuahua', 'Chinese-Crested', 'Chinese-Shar-Pei', 'Chinook', 'Chow-Chow', 'Cirneco-dell’Etna', 'Clumber-Spaniel', 'Cocker-Spaniel', 'Collie', 'Coton-de-Tulear', 'Croatian-Sheepdog', 'Curly-Coated-Retriever', 'Czechoslovakian-Vlciak', 'Dachshund', 'Dalmatian', 'Dandie-Dinmont-Terrier', 'Danish-Swedish-Farmdog', 'Deutscher-Wachtelhund', 'Doberman-Pinscher', 'Dogo-Argentino', 'Dogue-de-Bordeaux', 'Drentsche-Patrijshond', 'Drever', 'Dutch-Shepherd', 'English-Cocker-Spaniel', 'English-Foxhound', 'English-Setter', 'English-Springer-Spaniel', 'English-Toy-Spaniel', 'Entlebucher-Mountain-Dog', 'Estrela-Mountain-Dog', 'Eurasier', 'Field-Spaniel', 'Finnish-Lapphund', 'Finnish-Spitz', 'Flat-Coated-Retriever', 'French-Bulldog', 'French-Spaniel', 'German-Longhaired-Pointer', 'German-Pinscher', 'German-Shepherd-Dog', 'German-Shorthaired-Pointer', 'German-Spitz', 'German-Wirehaired-Pointer', 'Giant-Schnauzer', 'Glen-of-Imaal-Terrier', 'Golden-Retriever', 'Gordon-Setter', 'Grand-Basset-Griffon-Vendéen', 'Great-Dane', 'Great-Pyrenees', 'Greater-Swiss-Mountain-Dog', 'Greyhound', 'Hamiltonstovare', 'Hanoverian-Scenthound', 'Harrier', 'Havanese', 'Hokkaido', 'Hovawart', 'Ibizan-Hound', 'Icelandic-Sheepdog', 'Irish-Red-and-White-Setter', 'Irish-Setter', 'Irish-Terrier', 'Irish-Water-Spaniel', 'Irish-Wolfhound', 'Italian-Greyhound', 'Jagdterrier', 'Japanese-Akitainu', 'Japanese-Chin', 'Japanese-Spitz', 'Japanese-Terrier', 'Kai-Ken', 'Karelian-Bear-Dog', 'Keeshond', 'Kerry-Blue-Terrier', 'Kishu-Ken', 'Komondor', 'Korean-Jindo-Dog', 'Kromfohrlander', 'Kuvasz', 'Labrador-Retriever', 'Lagotto-Romagnolo', 'Lakeland-Terrier', 'Lancashire-Heeler', 'Lapponian-Herder', 'Large-Munsterlander', 'Leonberger', 'Lhasa-Apso', 'Löwchen', 'Maltese', 'Manchester-Terrier-(Standard)', 'Manchester-Terrier-(Toy)', 'Mastiff', 'Miniature-American-Shepherd', 'Miniature-Bull-Terrier', 'Miniature-Pinscher', 'Miniature-Schnauzer', 'Mountain-Cur', 'Mudi', 'Neapolitan-Mastiff', 'Nederlandse-Kooikerhondje', 'Newfoundland', 'Norfolk-Terrier', 'Norrbottenspets', 'Norwegian-Buhund', 'Norwegian-Elkhound', 'Norwegian-Lundehund', 'Norwich-Terrier', 'Nova-Scotia-Duck-Tolling-Retriever', 'Old-English-Sheepdog', 'Otterhound', 'Papillon', 'Parson-Russell-Terrier', 'Pekingese', 'Pembroke-Welsh-Corgi', 'Peruvian-Inca-Orchid', 'Petit-Basset-Griffon-Vendéen', 'Pharaoh-Hound', 'Plott-Hound', 'Pointer', 'Polish-Lowland-Sheepdog', 'Pomeranian', 'Pont-Audemer-Spaniel', 'Poodle-(Miniature)', 'Poodle-(Standard)', 'Poodle-(Toy)', 'Porcelaine', 'Portuguese-Podengo', 'Portuguese-Podengo-Pequeno', 'Portuguese-Pointer', 'Portuguese-Sheepdog', 'Portuguese-Water-Dog', 'Presa-Canario', 'Pudelpointer', 'Pug', 'Puli', 'Pumi', 'Pyrenean-Mastiff', 'Pyrenean-Shepherd', 'Rafeiro-do-Alentejo', 'Rat-Terrier', 'Redbone-Coonhound', 'Rhodesian-Ridgeback', 'Romanian-Carpathian-Shepherd', 'Romanian-Mioritic-Shepherd-Dog', 'Rottweiler', 'Russell-Terrier', 'Russian-Toy', 'Russian-Tsvetnaya-Bolonka', 'Saint-Bernard', 'Saluki', 'Samoyed', 'Schapendoes', 'Schipperke', 'Scottish-Deerhound', 'Scottish-Terrier', 'Sealyham-Terrier', 'Segugio-Italiano', 'Shetland-Sheepdog', 'Shiba-Inu', 'Shih-Tzu', 'Shikoku', 'Siberian-Husky', 'Silky-Terrier', 'Skye-Terrier', 'Sloughi', 'Slovakian-Wirehaired-Pointer', 'Slovensky-Cuvac', 'Slovensky-Kopov', 'Small-Munsterlander', 'Smooth-Fox-Terrier', 'Soft-Coated-Wheaten-Terrier', 'Spanish-Mastiff', 'Spanish-Water-Dog', 'Spinone-Italiano', 'Stabyhoun', 'Staffordshire-Bull-Terrier', 'Standard-Schnauzer', 'Sussex-Spaniel', 'Swedish-Lapphund', 'Swedish-Vallhund', 'Taiwan-Dog', 'Teddy-Roosevelt-Terrier', 'Thai-Bangkaew', 'Thai-Ridgeback', 'Tibetan-Mastiff', 'Tibetan-Spaniel', 'Tibetan-Terrier', 'Tornjak', 'Tosa', 'Toy-Fox-Terrier', 'Transylvanian-Hound', 'Treeing-Tennessee-Brindle', 'Treeing-Walker-Coonhound', 'Vizsla', 'Volpino-Italiano', 'Weimaraner', 'Welsh-Springer-Spaniel', 'Welsh-Terrier', 'West-Highland-White-Terrier', 'Wetterhoun', 'Whippet', 'Wire-Fox-Terrier', 'Wirehaired-Pointing-Griffon', 'Wirehaired-Vizsla', 'Working-Kelpie', 'Xoloitzcuintli', 'Yakutian-Laika', 'Yorkshire-Terrier']\n"
          ]
        }
      ],
      "source": [
        "#dog_name_scrapping\n",
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "\n",
        "headers_list = [{\"User-Agent\": 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36'},\n",
        "{\"User-Agent\": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246'},\n",
        "{\"User-Agent\": 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36'},\n",
        "{\"User-Agent\": 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9'},\n",
        " {\"User-Agent\": 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1'},\n",
        " {\"User-Agent\" : 'Mozilla/5.0 (X11; CrOS x86_64 8172.45.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.64 Safari/537.36'}]\n",
        "name_list = []\n",
        "\n",
        "def name_extract(page):\n",
        "    # page 1 ~ 23\n",
        "    url = f'https://www.akc.org/dog-breeds/page/{page}/'\n",
        "    r = requests.get(url, headers= headers_list[(page)%6])\n",
        "    soup = BeautifulSoup(r.content, 'html.parser')\n",
        "    divs = soup.find_all(class_ = \"breed-type-card__title mt0 mb0 f-25 py3 px3\")\n",
        "    for div in divs:\n",
        "        name_list.append(div.text.replace(\" \",\"-\")) # for the website we need to have '-' on url\n",
        "\n",
        "def main():\n",
        "    for i in range(1,25):\n",
        "        name_extract(i)\n",
        "    return name_list\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    name_list = main()\n",
        "    print(name_list)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#dog_name_cleaning\n",
        "import string\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "name_array = ['Affenpinscher', 'Afghan-Hound', 'Airedale-Terrier', 'Akita', 'Alaskan-Klee-Kai', 'Alaskan-Malamute', 'American-Bulldog', 'American-English-Coonhound', 'American-Eskimo-Dog', 'American-Foxhound', 'American-Hairless-Terrier', 'American-Leopard-Hound', 'American-Staffordshire-Terrier', 'American-Water-Spaniel', 'Anatolian-Shepherd-Dog', 'Appenzeller-Sennenhund', 'Australian-Cattle-Dog', 'Australian-Kelpie', 'Australian-Shepherd', 'Australian-Stumpy-Tail-Cattle-Dog', 'Australian-Terrier', 'Azawakh', 'Barbado-da-Terceira', 'Barbet', 'Basenji', 'Basset-Fauve-de-Bretagne', 'Basset-Hound', 'Bavarian-Mountain-Scent-Hound', 'Beagle', 'Bearded-Collie', 'Beauceron', 'Bedlington-Terrier', 'Belgian-Laekenois', 'Belgian-Malinois', 'Belgian-Sheepdog', 'Belgian-Tervuren', 'Bergamasco-Sheepdog', 'Berger-Picard', 'Bernese-Mountain-Dog', 'Bichon-Frise', 'Biewer-Terrier', 'Black-and-Tan-Coonhound', 'Black-Russian-Terrier', 'Bloodhound', 'Bluetick-Coonhound', 'Boerboel', 'Bohemian-Shepherd', 'Bolognese', 'Border-Collie', 'Border-Terrier', 'Borzoi', 'Boston-Terrier', 'Bouvier-des-Flandres', 'Boxer', 'Boykin-Spaniel', 'Bracco-Italiano', 'Braque-du-Bourbonnais', 'Braque-Francais-Pyrenean', 'Briard', 'Brittany', 'Broholmer', 'Brussels-Griffon', 'Bull-Terrier', 'Bulldog', 'Bullmastiff', 'Cairn-Terrier', 'Canaan-Dog', 'Cane-Corso', 'Cardigan-Welsh-Corgi', 'Carolina-Dog', 'Catahoula-Leopard-Dog', 'Caucasian-Shepherd-Dog', 'Cavalier-King-Charles-Spaniel', 'Central-Asian-Shepherd-Dog', 'Cesky-Terrier', 'Chesapeake-Bay-Retriever', 'Chihuahua', 'Chinese-Crested', 'Chinese-Shar-Pei', 'Chinook', 'Chow-Chow', 'Cirneco-dell’Etna', 'Clumber-Spaniel', 'Cocker-Spaniel', 'Collie', 'Coton-de-Tulear', 'Croatian-Sheepdog', 'Curly-Coated-Retriever', 'Czechoslovakian-Vlcak', 'Dachshund', 'Dalmatian', 'Dandie-Dinmont-Terrier', 'Danish-Swedish-Farmdog', 'Deutscher-Wachtelhund', 'Doberman-Pinscher', 'Dogo-Argentino', 'Dogue-de-Bordeaux', 'Drentsche-Patrijshond', 'Drever', 'Dutch-Shepherd', 'English-Cocker-Spaniel', 'English-Foxhound', 'English-Setter', 'English-Springer-Spaniel', 'English-Toy-Spaniel', 'Entlebucher-Mountain-Dog', 'Estrela-Mountain-Dog', 'Eurasier', 'Field-Spaniel', 'Finnish-Lapphund', 'Finnish-Spitz', 'Flat-Coated-Retriever', 'French-Bulldog', 'French-Spaniel', 'German-Longhaired-Pointer', 'German-Pinscher', 'German-Shepherd-Dog', 'German-Shorthaired-Pointer', 'German-Spitz', 'German-Wirehaired-Pointer', 'Giant-Schnauzer', 'Glen-of-Imaal-Terrier', 'Golden-Retriever', 'Gordon-Setter', 'Grand-Basset-Griffon-Vendéen', 'Great-Dane', 'Great-Pyrenees', 'Greater-Swiss-Mountain-Dog', 'Greyhound', 'Hamiltonstovare', 'Hanoverian-Scenthound', 'Harrier', 'Havanese', 'Hokkaido', 'Hovawart', 'Ibizan-Hound', 'Icelandic-Sheepdog', 'Irish-Red-and-White-Setter', 'Irish-Setter', 'Irish-Terrier', 'Irish-Water-Spaniel', 'Irish-Wolfhound', 'Italian-Greyhound', 'Jagdterrier', 'Japanese-Akitainu', 'Japanese-Chin', 'Japanese-Spitz', 'Japanese-Terrier', 'Jindo', 'Kai-Ken', 'Karelian-Bear-Dog', 'Keeshond', 'Kerry-Blue-Terrier', 'Kishu-Ken', 'Komondor', 'Kromfohrlander', 'Kuvasz', 'Labrador-Retriever', 'Lagotto-Romagnolo', 'Lakeland-Terrier', 'Lancashire-Heeler', 'Lapponian-Herder', 'Leonberger', 'Lhasa-Apso', 'Löwchen', 'Maltese', 'Manchester-Terrier-(Standard)', 'Manchester-Terrier-(Toy)', 'Mastiff', 'Miniature-American-Shepherd', 'Miniature-Bull-Terrier', 'Miniature-Pinscher', 'Miniature-Schnauzer', 'Mountain-Cur', 'Mudi', 'Neapolitan-Mastiff', 'Nederlandse-Kooikerhondje', 'Newfoundland', 'Norfolk-Terrier', 'Norrbottenspets', 'Norwegian-Buhund', 'Norwegian-Elkhound', 'Norwegian-Lundehund', 'Norwich-Terrier', 'Nova-Scotia-Duck-Tolling-Retriever', 'Old-English-Sheepdog', 'Otterhound', 'Papillon', 'Parson-Russell-Terrier', 'Pekingese', 'Pembroke-Welsh-Corgi', 'Perro-de-Presa-Canario', 'Peruvian-Inca-Orchid', 'Petit-Basset-Griffon-Vendéen', 'Pharaoh-Hound', 'Plott-Hound', 'Pointer', 'Polish-Lowland-Sheepdog', 'Pomeranian', 'Poodle-(Miniature)', 'Poodle-(Standard)', 'Poodle-(Toy)', 'Porcelaine', 'Portuguese-Podengo', 'Portuguese-Podengo-Pequeno', 'Portuguese-Pointer', 'Portuguese-Sheepdog', 'Portuguese-Water-Dog', 'Pudelpointer', 'Pug', 'Puli', 'Pumi', 'Pyrenean-Mastiff', 'Pyrenean-Shepherd', 'Rafeiro-do-Alentejo', 'Rat-Terrier', 'Redbone-Coonhound', 'Rhodesian-Ridgeback', 'Romanian-Carpathian-Shepherd', 'Romanian-Mioritic-Shepherd-Dog', 'Rottweiler', 'Russell-Terrier', 'Russian-Toy', 'Russian-Tsvetnaya-Bolonka', 'Saint-Bernard', 'Saluki', 'Samoyed', 'Schapendoes', 'Schipperke', 'Scottish-Deerhound', 'Scottish-Terrier', 'Sealyham-Terrier', 'Segugio-Italiano', 'Shetland-Sheepdog', 'Shiba-Inu', 'Shih-Tzu', 'Shikoku', 'Siberian-Husky', 'Silky-Terrier', 'Skye-Terrier', 'Sloughi', 'Slovakian-Wirehaired-Pointer', 'Slovensky-Cuvac', 'Slovensky-Kopov', 'Small-Munsterlander', 'Smooth-Fox-Terrier', 'Soft-Coated-Wheaten-Terrier', 'Spanish-Mastiff', 'Spanish-Water-Dog', 'Spinone-Italiano', 'Stabyhoun', 'Staffordshire-Bull-Terrier', 'Standard-Schnauzer', 'Sussex-Spaniel', 'Swedish-Lapphund', 'Swedish-Vallhund', 'Taiwan-Dog', 'Teddy-Roosevelt-Terrier', 'Thai-Ridgeback', 'Tibetan-Mastiff', 'Tibetan-Spaniel', 'Tibetan-Terrier', 'Tornjak', 'Tosa', 'Toy-Fox-Terrier', 'Transylvanian-Hound', 'Treeing-Tennessee-Brindle', 'Treeing-Walker-Coonhound', 'Vizsla', 'Volpino-Italiano', 'Weimaraner', 'Welsh-Springer-Spaniel', 'Welsh-Terrier', 'West-Highland-White-Terrier', 'Wetterhoun', 'Whippet', 'Wire-Fox-Terrier', 'Wirehaired-Pointing-Griffon', 'Wirehaired-Vizsla', 'Working-Kelpie', 'Xoloitzcuintli', 'Yakutian-Laika', 'Yorkshire-Terrier']\n",
        "\n",
        "def print_non_alphabet_name():\n",
        "    name_with_non_alphabet = []\n",
        "    for name in name_array:\n",
        "        for char in name:\n",
        "            if char not in string.ascii_letters + '-()':\n",
        "                name_with_non_alphabet.append(name)\n",
        "                print(name)\n",
        "                break\n",
        "\n",
        "# print_non_alphabet_name()\n",
        "\n",
        "edit_dict = {'Cirneco-dell’Etna' : 'cirneco-delletna',\n",
        "'Grand-Basset-Griffon-Vendéen' : 'Grand-Basset-Griffon-Vendeen',\n",
        "'Löwchen' : 'Lowchen',\n",
        "'Petit-Basset-Griffon-Vendéen' : 'Petit-Basset-Griffon-Vendeen'}\n",
        "\n",
        "# edit names with non_alphabet\n",
        "for i, name in enumerate(name_array):\n",
        "    if name in edit_dict:\n",
        "        name_array[i] = edit_dict[name]\n",
        "\n",
        "# lower all the names\n",
        "name_array =  [i.lower() for i in name_array]\n",
        "\n",
        "np.savetxt(\"dog_name.csv\",\n",
        "           name_array,\n",
        "           delimiter =\", \",\n",
        "           fmt ='% s')"
      ],
      "metadata": {
        "id": "Ks_sc-iPpg3M"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#dog_info_scrapping\n",
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "# from selenium.webdriver import Chrome\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import time\n",
        "import os\n",
        "\n",
        "def add_height_weight_life(soup,dog_info_dict):\n",
        "\n",
        "    h_w_life = soup.find_all(class_ = 'f-16 my0 lh-solid breed-page__hero__overview__subtitle')\n",
        "\n",
        "    # There is a case when no height, weight or life on the description\n",
        "    height = \"\"\n",
        "    weight = \"\"\n",
        "    life = \"\"\n",
        "\n",
        "    for i in h_w_life: # choose the first text, there will be multiple\n",
        "        if \"inches\" in i.text:\n",
        "            height = height + \"~\" + i.text\n",
        "        elif \"pounds\" in i.text:\n",
        "            weight = weight + \"~\" + i.text\n",
        "        elif \"years\" in i.text:\n",
        "            life = i.text\n",
        "\n",
        "    dog_info_dict['height'] = height\n",
        "    dog_info_dict['weight'] = weight\n",
        "    dog_info_dict['life'] = life\n",
        "\n",
        "    return dog_info_dict\n",
        "\n",
        "def add_traits(soup,dog_info_dict):\n",
        "    traits = soup.find_all(class_ = \"breed-trait-group__trait-all\")\n",
        "    for trait in traits:\n",
        "        trait_name = trait.find(class_ = \"breed-trait-group__header breed-trait-group__header-closed\")\n",
        "        score = str(trait).count(\"filled\")\n",
        "\n",
        "        if trait_name.text == 'Coat Type' or trait_name.text == 'Coat Length':\n",
        "            properties = trait.find_all(class_= \"breed-trait-score__choice--selected\")\n",
        "            result = \"\"\n",
        "            for property in properties:\n",
        "                if result == \"\":\n",
        "                    result = property.text\n",
        "                else:\n",
        "                    result = result + \"-\" + property.text\n",
        "            dog_info_dict[trait_name.text] = result\n",
        "        else:\n",
        "            dog_info_dict[trait_name.text] = score\n",
        "    return dog_info_dict\n",
        "\n",
        "def add_popular_rank(soup,dog_info_dict):\n",
        "    try:\n",
        "        popularity_rank = soup.find('span',class_ =  \"breed-page__popularity__custom-label\")\n",
        "        dog_info_dict[\"popularity_rank\"] = popularity_rank.text\n",
        "    except:\n",
        "        dog_info_dict[\"popularity_rank\"] = np.nan\n",
        "\n",
        "    return dog_info_dict\n",
        "\n",
        "def add_color(soup,dog_info_dict):\n",
        "    # color info\n",
        "    colors_table = soup.find(class_ =  \"breed-standard__table\")\n",
        "    colors = colors_table.find_all(class_ = \"accordion-toggle\")\n",
        "    color_result = \"\"\n",
        "    for color in colors:\n",
        "        if color_result == \"\":\n",
        "            color_result = color.find('td').text\n",
        "        else:\n",
        "            color_result = color_result + \"-\" + color.find('td').text\n",
        "    dog_info_dict[\"color\"] = color_result\n",
        "    return dog_info_dict\n",
        "\n",
        "def add_mark(soup,dog_info_dict):\n",
        "    # marking, no marking exist\n",
        "    marking_result = \"\"\n",
        "    marking_table = soup.find(id =  \"markings-t-h\").find('tbody')\n",
        "    try:\n",
        "        marking_rows = marking_table.find_all('tr')\n",
        "        marking_result = \"\"\n",
        "        for marking in marking_rows:\n",
        "            if marking_result == \"\":\n",
        "                marking_result = marking.find('td').text\n",
        "            else:\n",
        "                marking_result = marking_result + \"-\" + marking.find('td').text\n",
        "        dog_info_dict[\"marking\"] = marking_result\n",
        "    except:\n",
        "        dog_info_dict[\"marking\"] = np.nan\n",
        "    return dog_info_dict\n",
        "\n",
        "def add_breed_info(soup, dog_info_dict):\n",
        "\n",
        "    # breed info\n",
        "    breed_info = soup.find(class_ =  \"breed-page__about__read-more__text\")\n",
        "    dog_info_dict[\"breed_info\"] = breed_info.text\n",
        "\n",
        "    return dog_info_dict\n",
        "\n",
        "def add_last_info(soup,dog_info_dict):\n",
        "    # health, grooming, excercise, training, nutrition\n",
        "    last_info = soup.find_all(class_ = \"breed-table__accordion-padding__p\")\n",
        "    dog_info_dict[\"health\"] = last_info[0].text\n",
        "    dog_info_dict[\"grooming\"] = last_info[1].text\n",
        "    dog_info_dict[\"excercise\"] = last_info[2].text\n",
        "    dog_info_dict[\"training\"] = last_info[3].text\n",
        "    dog_info_dict[\"nutrition\"] = last_info[4].text\n",
        "    return dog_info_dict\n",
        "\n",
        "\n",
        "def extract(dog_name,driver):\n",
        "    # instatiate dog info dictionary\n",
        "    dog_info_dict = {'dog' : dog_name}\n",
        "\n",
        "    url = f'https://www.akc.org/dog-breeds/{dog_name}/'\n",
        "    # driver = Chrome(executable_path= '/Users/glee2/Downloads/chromedriver')\n",
        "    driver.get(url)\n",
        "\n",
        "    soup = BeautifulSoup(driver.page_source, 'html.parser')\n",
        "\n",
        "    # add info to dog_info_dict\n",
        "    dog_info_dict = add_height_weight_life(soup,dog_info_dict)\n",
        "    dog_info_dict = add_traits(soup,dog_info_dict)\n",
        "    dog_info_dict = add_popular_rank(soup,dog_info_dict)\n",
        "    dog_info_dict = add_color(soup,dog_info_dict)\n",
        "    dog_info_dict = add_mark(soup,dog_info_dict)\n",
        "    dog_info_dict = add_last_info(soup,dog_info_dict) # health, grooming, excercise, training, nutrition\n",
        "\n",
        "    # driver.quit()\n",
        "    return dog_info_dict\n",
        "\n",
        "# def main():\n",
        "#     dog_name_series = pd.read_csv('dog_name.csv', header= None)\n",
        "#     dog_name_list = [name[0] for name in dog_name_series.values]\n",
        "#     dog_data_dict = []\n",
        "\n",
        "#     # driver = Chrome(executable_path= '/Users/glee2/Downloads/chromedriver')\n",
        "#     # os.environ[\"PATH\"] += os.pathsep + '/Users/glee2/Downloads/'\n",
        "#     # driver = Chrome()\n",
        "\n",
        "# # Create the WebDriver instance\n",
        "#     # driver = Chrome()\n",
        "\n",
        "#     for i in range(len(dog_name_list)):\n",
        "#         print(f\"{i}th dog {dog_name_list[i]}\")\n",
        "#         dog_data_dict.append(extract(dog_name_list[i]))\n",
        "\n",
        "#     # driver.quit()\n",
        "#     dog_data_df = pd.DataFrame(dog_data_dict)\n",
        "#     dog_data_df.to_csv(\"dog_data_09032022.csv\")\n",
        "#     print(\"done\")\n",
        "\n",
        "\n",
        "def main():\n",
        "    dog_name_series = pd.read_csv('dog_name.csv', header=None)\n",
        "    dog_name_list = [name[0] for name in dog_name_series.values]\n",
        "    dog_data_dict = []\n",
        "\n",
        "    for i in range(len(dog_name_list)):\n",
        "        print(f\"{i}th dog {dog_name_list[i]}\")\n",
        "        dog_data_dict.append(extract(dog_name_list[i]))\n",
        "\n",
        "    dog_data_df = pd.DataFrame(dog_data_dict)\n",
        "    dog_data_df.to_csv(\"dog_data_09032022.csv\")\n",
        "    print(\"done\")\n",
        "\n",
        "def extract(dog_name):\n",
        "    # Define your extraction logic here\n",
        "    pass\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    main()\n",
        ""
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sgFCCQ4iptlw",
        "outputId": "b5a2896d-6cc8-4572-f3b5-c83e7a108e34"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0th dog affenpinscher\n",
            "1th dog afghan-hound\n",
            "2th dog airedale-terrier\n",
            "3th dog akita\n",
            "4th dog alaskan-klee-kai\n",
            "5th dog alaskan-malamute\n",
            "6th dog american-bulldog\n",
            "7th dog american-english-coonhound\n",
            "8th dog american-eskimo-dog\n",
            "9th dog american-foxhound\n",
            "10th dog american-hairless-terrier\n",
            "11th dog american-leopard-hound\n",
            "12th dog american-staffordshire-terrier\n",
            "13th dog american-water-spaniel\n",
            "14th dog anatolian-shepherd-dog\n",
            "15th dog appenzeller-sennenhund\n",
            "16th dog australian-cattle-dog\n",
            "17th dog australian-kelpie\n",
            "18th dog australian-shepherd\n",
            "19th dog australian-stumpy-tail-cattle-dog\n",
            "20th dog australian-terrier\n",
            "21th dog azawakh\n",
            "22th dog barbado-da-terceira\n",
            "23th dog barbet\n",
            "24th dog basenji\n",
            "25th dog basset-fauve-de-bretagne\n",
            "26th dog basset-hound\n",
            "27th dog bavarian-mountain-scent-hound\n",
            "28th dog beagle\n",
            "29th dog bearded-collie\n",
            "30th dog beauceron\n",
            "31th dog bedlington-terrier\n",
            "32th dog belgian-laekenois\n",
            "33th dog belgian-malinois\n",
            "34th dog belgian-sheepdog\n",
            "35th dog belgian-tervuren\n",
            "36th dog bergamasco-sheepdog\n",
            "37th dog berger-picard\n",
            "38th dog bernese-mountain-dog\n",
            "39th dog bichon-frise\n",
            "40th dog biewer-terrier\n",
            "41th dog black-and-tan-coonhound\n",
            "42th dog black-russian-terrier\n",
            "43th dog bloodhound\n",
            "44th dog bluetick-coonhound\n",
            "45th dog boerboel\n",
            "46th dog bohemian-shepherd\n",
            "47th dog bolognese\n",
            "48th dog border-collie\n",
            "49th dog border-terrier\n",
            "50th dog borzoi\n",
            "51th dog boston-terrier\n",
            "52th dog bouvier-des-flandres\n",
            "53th dog boxer\n",
            "54th dog boykin-spaniel\n",
            "55th dog bracco-italiano\n",
            "56th dog braque-du-bourbonnais\n",
            "57th dog braque-francais-pyrenean\n",
            "58th dog briard\n",
            "59th dog brittany\n",
            "60th dog broholmer\n",
            "61th dog brussels-griffon\n",
            "62th dog bull-terrier\n",
            "63th dog bulldog\n",
            "64th dog bullmastiff\n",
            "65th dog cairn-terrier\n",
            "66th dog canaan-dog\n",
            "67th dog cane-corso\n",
            "68th dog cardigan-welsh-corgi\n",
            "69th dog carolina-dog\n",
            "70th dog catahoula-leopard-dog\n",
            "71th dog caucasian-shepherd-dog\n",
            "72th dog cavalier-king-charles-spaniel\n",
            "73th dog central-asian-shepherd-dog\n",
            "74th dog cesky-terrier\n",
            "75th dog chesapeake-bay-retriever\n",
            "76th dog chihuahua\n",
            "77th dog chinese-crested\n",
            "78th dog chinese-shar-pei\n",
            "79th dog chinook\n",
            "80th dog chow-chow\n",
            "81th dog cirneco-delletna\n",
            "82th dog clumber-spaniel\n",
            "83th dog cocker-spaniel\n",
            "84th dog collie\n",
            "85th dog coton-de-tulear\n",
            "86th dog croatian-sheepdog\n",
            "87th dog curly-coated-retriever\n",
            "88th dog czechoslovakian-vlcak\n",
            "89th dog dachshund\n",
            "90th dog dalmatian\n",
            "91th dog dandie-dinmont-terrier\n",
            "92th dog danish-swedish-farmdog\n",
            "93th dog deutscher-wachtelhund\n",
            "94th dog doberman-pinscher\n",
            "95th dog dogo-argentino\n",
            "96th dog dogue-de-bordeaux\n",
            "97th dog drentsche-patrijshond\n",
            "98th dog drever\n",
            "99th dog dutch-shepherd\n",
            "100th dog english-cocker-spaniel\n",
            "101th dog english-foxhound\n",
            "102th dog english-setter\n",
            "103th dog english-springer-spaniel\n",
            "104th dog english-toy-spaniel\n",
            "105th dog entlebucher-mountain-dog\n",
            "106th dog estrela-mountain-dog\n",
            "107th dog eurasier\n",
            "108th dog field-spaniel\n",
            "109th dog finnish-lapphund\n",
            "110th dog finnish-spitz\n",
            "111th dog flat-coated-retriever\n",
            "112th dog french-bulldog\n",
            "113th dog french-spaniel\n",
            "114th dog german-longhaired-pointer\n",
            "115th dog german-pinscher\n",
            "116th dog german-shepherd-dog\n",
            "117th dog german-shorthaired-pointer\n",
            "118th dog german-spitz\n",
            "119th dog german-wirehaired-pointer\n",
            "120th dog giant-schnauzer\n",
            "121th dog glen-of-imaal-terrier\n",
            "122th dog golden-retriever\n",
            "123th dog gordon-setter\n",
            "124th dog grand-basset-griffon-vendeen\n",
            "125th dog great-dane\n",
            "126th dog great-pyrenees\n",
            "127th dog greater-swiss-mountain-dog\n",
            "128th dog greyhound\n",
            "129th dog hamiltonstovare\n",
            "130th dog hanoverian-scenthound\n",
            "131th dog harrier\n",
            "132th dog havanese\n",
            "133th dog hokkaido\n",
            "134th dog hovawart\n",
            "135th dog ibizan-hound\n",
            "136th dog icelandic-sheepdog\n",
            "137th dog irish-red-and-white-setter\n",
            "138th dog irish-setter\n",
            "139th dog irish-terrier\n",
            "140th dog irish-water-spaniel\n",
            "141th dog irish-wolfhound\n",
            "142th dog italian-greyhound\n",
            "143th dog jagdterrier\n",
            "144th dog japanese-akitainu\n",
            "145th dog japanese-chin\n",
            "146th dog japanese-spitz\n",
            "147th dog japanese-terrier\n",
            "148th dog jindo\n",
            "149th dog kai-ken\n",
            "150th dog karelian-bear-dog\n",
            "151th dog keeshond\n",
            "152th dog kerry-blue-terrier\n",
            "153th dog kishu-ken\n",
            "154th dog komondor\n",
            "155th dog kromfohrlander\n",
            "156th dog kuvasz\n",
            "157th dog labrador-retriever\n",
            "158th dog lagotto-romagnolo\n",
            "159th dog lakeland-terrier\n",
            "160th dog lancashire-heeler\n",
            "161th dog lapponian-herder\n",
            "162th dog leonberger\n",
            "163th dog lhasa-apso\n",
            "164th dog lowchen\n",
            "165th dog maltese\n",
            "166th dog manchester-terrier-(standard)\n",
            "167th dog manchester-terrier-(toy)\n",
            "168th dog mastiff\n",
            "169th dog miniature-american-shepherd\n",
            "170th dog miniature-bull-terrier\n",
            "171th dog miniature-pinscher\n",
            "172th dog miniature-schnauzer\n",
            "173th dog mountain-cur\n",
            "174th dog mudi\n",
            "175th dog neapolitan-mastiff\n",
            "176th dog nederlandse-kooikerhondje\n",
            "177th dog newfoundland\n",
            "178th dog norfolk-terrier\n",
            "179th dog norrbottenspets\n",
            "180th dog norwegian-buhund\n",
            "181th dog norwegian-elkhound\n",
            "182th dog norwegian-lundehund\n",
            "183th dog norwich-terrier\n",
            "184th dog nova-scotia-duck-tolling-retriever\n",
            "185th dog old-english-sheepdog\n",
            "186th dog otterhound\n",
            "187th dog papillon\n",
            "188th dog parson-russell-terrier\n",
            "189th dog pekingese\n",
            "190th dog pembroke-welsh-corgi\n",
            "191th dog perro-de-presa-canario\n",
            "192th dog peruvian-inca-orchid\n",
            "193th dog petit-basset-griffon-vendeen\n",
            "194th dog pharaoh-hound\n",
            "195th dog plott-hound\n",
            "196th dog pointer\n",
            "197th dog polish-lowland-sheepdog\n",
            "198th dog pomeranian\n",
            "199th dog poodle-(miniature)\n",
            "200th dog poodle-(standard)\n",
            "201th dog poodle-(toy)\n",
            "202th dog porcelaine\n",
            "203th dog portuguese-podengo\n",
            "204th dog portuguese-podengo-pequeno\n",
            "205th dog portuguese-pointer\n",
            "206th dog portuguese-sheepdog\n",
            "207th dog portuguese-water-dog\n",
            "208th dog pudelpointer\n",
            "209th dog pug\n",
            "210th dog puli\n",
            "211th dog pumi\n",
            "212th dog pyrenean-mastiff\n",
            "213th dog pyrenean-shepherd\n",
            "214th dog rafeiro-do-alentejo\n",
            "215th dog rat-terrier\n",
            "216th dog redbone-coonhound\n",
            "217th dog rhodesian-ridgeback\n",
            "218th dog romanian-carpathian-shepherd\n",
            "219th dog romanian-mioritic-shepherd-dog\n",
            "220th dog rottweiler\n",
            "221th dog russell-terrier\n",
            "222th dog russian-toy\n",
            "223th dog russian-tsvetnaya-bolonka\n",
            "224th dog saint-bernard\n",
            "225th dog saluki\n",
            "226th dog samoyed\n",
            "227th dog schapendoes\n",
            "228th dog schipperke\n",
            "229th dog scottish-deerhound\n",
            "230th dog scottish-terrier\n",
            "231th dog sealyham-terrier\n",
            "232th dog segugio-italiano\n",
            "233th dog shetland-sheepdog\n",
            "234th dog shiba-inu\n",
            "235th dog shih-tzu\n",
            "236th dog shikoku\n",
            "237th dog siberian-husky\n",
            "238th dog silky-terrier\n",
            "239th dog skye-terrier\n",
            "240th dog sloughi\n",
            "241th dog slovakian-wirehaired-pointer\n",
            "242th dog slovensky-cuvac\n",
            "243th dog slovensky-kopov\n",
            "244th dog small-munsterlander\n",
            "245th dog smooth-fox-terrier\n",
            "246th dog soft-coated-wheaten-terrier\n",
            "247th dog spanish-mastiff\n",
            "248th dog spanish-water-dog\n",
            "249th dog spinone-italiano\n",
            "250th dog stabyhoun\n",
            "251th dog staffordshire-bull-terrier\n",
            "252th dog standard-schnauzer\n",
            "253th dog sussex-spaniel\n",
            "254th dog swedish-lapphund\n",
            "255th dog swedish-vallhund\n",
            "256th dog taiwan-dog\n",
            "257th dog teddy-roosevelt-terrier\n",
            "258th dog thai-ridgeback\n",
            "259th dog tibetan-mastiff\n",
            "260th dog tibetan-spaniel\n",
            "261th dog tibetan-terrier\n",
            "262th dog tornjak\n",
            "263th dog tosa\n",
            "264th dog toy-fox-terrier\n",
            "265th dog transylvanian-hound\n",
            "266th dog treeing-tennessee-brindle\n",
            "267th dog treeing-walker-coonhound\n",
            "268th dog vizsla\n",
            "269th dog volpino-italiano\n",
            "270th dog weimaraner\n",
            "271th dog welsh-springer-spaniel\n",
            "272th dog welsh-terrier\n",
            "273th dog west-highland-white-terrier\n",
            "274th dog wetterhoun\n",
            "275th dog whippet\n",
            "276th dog wire-fox-terrier\n",
            "277th dog wirehaired-pointing-griffon\n",
            "278th dog wirehaired-vizsla\n",
            "279th dog working-kelpie\n",
            "280th dog xoloitzcuintli\n",
            "281th dog yakutian-laika\n",
            "282th dog yorkshire-terrier\n",
            "done\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "WuTy-UEQqukk"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}