Add support for CrateDB to LangChain LLM framework

docs/docs/.gitignore

@@ -4,4 +4,5 @@ node_modules/
 
 .docusaurus
 .cache-loader
-docs/api
+docs/api
+example.sqlite

docs/docs/integrations/document_loaders/cratedb.ipynb

@@ -0,0 +1,232 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# CrateDB\n",
+    "\n",
+    "This notebook demonstrates how to load documents from a [CrateDB] database,\n",
+    "using the [SQLAlchemy] document loader.\n",
+    "\n",
+    "It loads the result of a database query with one document per row.\n",
+    "\n",
+    "[CrateDB]: https://github.com/crate/crate\n",
+    "[SQLAlchemy]: https://www.sqlalchemy.org/"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "## Prerequisites"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "#!pip install crash langchain sqlalchemy-cratedb"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "Populate database."
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001B[32mCONNECT OK\r\n",
+      "\u001B[0m\u001B[32mPSQL OK, 1 row affected (0.001 sec)\r\n",
+      "\u001B[0m\u001B[32mDELETE OK, 30 rows affected (0.008 sec)\r\n",
+      "\u001B[0m\u001B[32mINSERT OK, 30 rows affected (0.011 sec)\r\n",
+      "\u001B[0m\u001B[0m\u001B[32mCONNECT OK\r\n",
+      "\u001B[0m\u001B[32mREFRESH OK, 1 row affected (0.001 sec)\r\n",
+      "\u001B[0m\u001B[0m"
+     ]
+    }
+   ],
+   "source": [
+    "!crash < ./example_data/mlb_teams_2012.sql\n",
+    "!crash --command \"REFRESH TABLE mlb_teams_2012;\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "## Usage"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 42,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langchain.document_loaders import CrateDBLoader\n",
+    "from pprint import pprint\n",
+    "\n",
+    "CONNECTION_STRING = \"crate://crate@localhost/\"\n",
+    "\n",
+    "loader = CrateDBLoader(\n",
+    "    'SELECT * FROM mlb_teams_2012 ORDER BY \"Team\" LIMIT 5;',\n",
+    "    url=CONNECTION_STRING,\n",
+    ")\n",
+    "documents = loader.load()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[Document(page_content='Team: Angels\\nPayroll (millions): 154.49\\nWins: 89', metadata={}),\n",
+      " Document(page_content='Team: Astros\\nPayroll (millions): 60.65\\nWins: 55', metadata={}),\n",
+      " Document(page_content='Team: Athletics\\nPayroll (millions): 55.37\\nWins: 94', metadata={}),\n",
+      " Document(page_content='Team: Blue Jays\\nPayroll (millions): 75.48\\nWins: 73', metadata={}),\n",
+      " Document(page_content='Team: Braves\\nPayroll (millions): 83.31\\nWins: 94', metadata={})]\n"
+     ]
+    }
+   ],
+   "source": [
+    "pprint(documents)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Specifying Which Columns are Content vs Metadata"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loader = CrateDBLoader(\n",
+    "    'SELECT * FROM mlb_teams_2012 ORDER BY \"Team\" LIMIT 5;',\n",
+    "    url=CONNECTION_STRING,\n",
+    "    page_content_columns=[\"Team\"],\n",
+    "    metadata_columns=[\"Payroll (millions)\"],\n",
+    ")\n",
+    "documents = loader.load()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[Document(page_content='Team: Angels', metadata={'Payroll (millions)': 154.49}),\n",
+      " Document(page_content='Team: Astros', metadata={'Payroll (millions)': 60.65}),\n",
+      " Document(page_content='Team: Athletics', metadata={'Payroll (millions)': 55.37}),\n",
+      " Document(page_content='Team: Blue Jays', metadata={'Payroll (millions)': 75.48}),\n",
+      " Document(page_content='Team: Braves', metadata={'Payroll (millions)': 83.31})]\n"
+     ]
+    }
+   ],
+   "source": [
+    "pprint(documents)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Adding Source to Metadata"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loader = CrateDBLoader(\n",
+    "    'SELECT * FROM mlb_teams_2012 ORDER BY \"Team\" LIMIT 5;',\n",
+    "    url=CONNECTION_STRING,\n",
+    "    source_columns=[\"Team\"],\n",
+    ")\n",
+    "documents = loader.load()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 47,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[Document(page_content='Team: Angels\\nPayroll (millions): 154.49\\nWins: 89', metadata={'source': 'Angels'}),\n",
+      " Document(page_content='Team: Astros\\nPayroll (millions): 60.65\\nWins: 55', metadata={'source': 'Astros'}),\n",
+      " Document(page_content='Team: Athletics\\nPayroll (millions): 55.37\\nWins: 94', metadata={'source': 'Athletics'}),\n",
+      " Document(page_content='Team: Blue Jays\\nPayroll (millions): 75.48\\nWins: 73', metadata={'source': 'Blue Jays'}),\n",
+      " Document(page_content='Team: Braves\\nPayroll (millions): 83.31\\nWins: 94', metadata={'source': 'Braves'})]\n"
+     ]
+    }
+   ],
+   "source": [
+    "pprint(documents)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

docs/docs/integrations/document_loaders/example_data/mlb_teams_2012.sql

@@ -1,5 +1,6 @@
 -- Provisioning table "mlb_teams_2012".
 --
+-- crash < mlb_teams_2012.sql
 -- psql postgresql://postgres@localhost < mlb_teams_2012.sql
 
 DROP TABLE IF EXISTS mlb_teams_2012;