拓展-langchain网络爬虫.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "09ad2c20-cfde-4d7e-8d6a-1fd27ffc726a",
   "metadata": {},
   "source": [
    "# 介绍langchain的网络爬虫 Apify"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c72cbffc-6090-45dd-960a-3d34ca9c7864",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name: apify_client\n",
      "Version: 1.6.2\n",
      "Summary: Apify API client for Python\n",
      "Home-page: \n",
      "Author: \n",
      "Author-email: \"Apify Technologies s.r.o.\" <support@apify.com>\n",
      "License: Apache Software License\n",
      "Location: d:\\chatbot\\jupyter\\.venv\\lib\\site-packages\n",
      "Requires: apify-shared, httpx\n",
      "Required-by: \n",
      "---\n",
      "Name: tiktoken\n",
      "Version: 0.5.2\n",
      "Summary: tiktoken is a fast BPE tokeniser for use with OpenAI's models\n",
      "Home-page: \n",
      "Author: Shantanu Jain\n",
      "Author-email: shantanu@openai.com\n",
      "License: MIT License\n",
      "        \n",
      "        Copyright (c) 2022 OpenAI, Shantanu Jain\n",
      "        \n",
      "        Permission is hereby granted, free of charge, to any person obtaining a copy\n",
      "        of this software and associated documentation files (the \"Software\"), to deal\n",
      "        in the Software without restriction, including without limitation the rights\n",
      "        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n",
      "        copies of the Software, and to permit persons to whom the Software is\n",
      "        furnished to do so, subject to the following conditions:\n",
      "        \n",
      "        The above copyright notice and this permission notice shall be included in all\n",
      "        copies or substantial portions of the Software.\n",
      "        \n",
      "        THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n",
      "        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n",
      "        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n",
      "        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n",
      "        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n",
      "        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n",
      "        SOFTWARE.\n",
      "        \n",
      "Location: d:\\chatbot\\jupyter\\.venv\\lib\\site-packages\n",
      "Requires: regex, requests\n",
      "Required-by: langchain-openai\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip show apify-client tiktoken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9185c1b6-02ec-43b7-a2a3-4ad1548718ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apify_api_D3jAaQkWFags9GSmzDjx2np7Js0kEJ1HEDXu\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "open_api_key = os.getenv(\"OPENAI_API_KEY\")\n",
    "apify_api_key = os.getenv(\"APIFY_API_TOKEN\")\n",
    "\n",
    "print(apify_api_key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3cd26864-2fdb-449f-8639-2b92b04081d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.docstore.document import Document\n",
    "from langchain.indexes import VectorstoreIndexCreator\n",
    "from langchain_community.utilities import ApifyWrapper\n",
    "\n",
    "apify = ApifyWrapper()\n",
    "\n",
    "# Call the Actor to obtain text from the crawled webpages\n",
    "loader = apify.call_actor(\n",
    "    actor_id =\"apify/website-content-crawler\",\n",
    "    run_input = {\n",
    "        \"startUrls\":[{\"url\":\"https://python.langchain.com/docs/integrations/chat/\"}]\n",
    "    },\n",
    "    dataset_mapping_function = lambda item:Document(\n",
    "        page_content = item[\"text\"] or \"\", metadata ={\"source\":item[\"url\"]}\n",
    "    ),\n",
    ")\n",
    "\n",
    "# Create a vector store based on the crawled data\n",
    "index = VectorstoreIndexCreator().from_loaders([loader])\n",
    "\n",
    "# Query the vector store\n",
    "query = \"Are any OpenAI chat models integrated in LangChain?\"\n",
    "result = index.query(query)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d4378ab-f5ef-4b06-9884-813ac0d692e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "object_size = sys.getsizeof(index)\n",
    "print(f\"Object Size: {object_size} bytes\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}