examples/output_formats_demonstration.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RedisTimeseriesManager Output Formats Demonstration\n",
    "\n",
    "See an example of supported output formats.\n",
    "\n",
    "Supported formats are 'list' (python `list`, default format), 'df'(pandas dataframe), 'dict' (dictioanry of lines), 'sets-list' (sets of data including lables and lists), 'sets-df' (sets of data  including labels and dataframes), 'sets-dict' (sets of data including labels and dictionary of lines) and 'raw' which is the raw data read from the timeseries."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialization\n",
    "from _path import add_root\n",
    "add_root(up_levels=1)\n",
    "\n",
    "from src.redis_timeseries_manager import RedisTimeseriesManager\n",
    "\n",
    "class RedisTimeseriesManagerTests(RedisTimeseriesManager):\n",
    "    _name = 'test1'\n",
    "    _lines = ['l1', 'l2']\n",
    "    _timeframes = {\n",
    "        'raw': {'retention_secs': 100000}, #unlimited\n",
    "    }\n",
    "\n",
    "default_test_settings = {\n",
    "    'host': 'localhost',\n",
    "    'port': 6379,\n",
    "    'db': 0,\n",
    "    'password': None,\n",
    "}\n",
    "try:\n",
    "    from settings import test_settings\n",
    "except:\n",
    "    test_settings = None\n",
    "test_settings = test_settings or default_test_settings\n",
    "\n",
    "tl = RedisTimeseriesManagerTests(**test_settings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set/reset test data\n",
    "def reset_test_data():\n",
    "    # clear previous data\n",
    "    keys = tl.query_index(return_key_names=True)\n",
    "    for key in keys[1]:\n",
    "        tl.client.delete(key)\n",
    "    # set test data\n",
    "    data1 = [\n",
    "        ['btc',100, 1, 2],\n",
    "        ['btc',101, 3, 4],\n",
    "        ['btc',102, 5, 6],\n",
    "        ['eth', 101, 7, 8],\n",
    "        ['eth', 102, 9, 10],\n",
    "        ['eth', 103, 11, 12],\n",
    "        ['eth', 104, 13, 14],\n",
    "    ]\n",
    "    tl.insert(\n",
    "        data=data1,\n",
    "        c1='crypto',\n",
    "        c2_position=0,\n",
    "        create_inplace=True,\n",
    "        extra_labels={'iscrypto': 'yes'}\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# preparing data\n",
    "reset_test_data()\n",
    "c1 = 'crypto'\n",
    "c2 = {\n",
    "    'iscrypto': 'yes'\n",
    "}"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generating read error by not allowing multiple data-points with the same time\n",
    "\n",
    "Allowing multiple data-points with the same time can possibly results in data ambiguity if not used with consideration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Inadequate filters: Multiple lines with the same name are returned. This generally happens when label filters are used as classifiers and labels are not enough. Please add all labels to filter out data properly or turn on the `allow_multiple` option'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "success, result = tl.read(\n",
    "    c1=c1,\n",
    "    c2=c2,\n",
    ")\n",
    "result"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Output as `list` format(default)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[100.0, 1.0, 2.0],\n",
       " [101.0, 3.0, 4.0],\n",
       " [101.0, 7.0, 8.0],\n",
       " [102.0, 5.0, 6.0],\n",
       " [102.0, 9.0, 10.0],\n",
       " [103.0, 11.0, 12.0],\n",
       " [104.0, 13.0, 14.0]]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "success, data = tl.read(\n",
    "    c1=c1,\n",
    "    c2=c2,\n",
    "    allow_multiple=True,\n",
    "    return_as='list'\n",
    ")\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl.get_read_length(data=data, data_type='list')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Output as `df`(pandas dataframe) format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>l1</th>\n",
       "      <th>l2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>101</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>101</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>102</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>102</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>103</td>\n",
       "      <td>11.0</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>104</td>\n",
       "      <td>13.0</td>\n",
       "      <td>14.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   time    l1    l2\n",
       "0   100   1.0   2.0\n",
       "1   101   3.0   4.0\n",
       "2   101   7.0   8.0\n",
       "3   102   5.0   6.0\n",
       "4   102   9.0  10.0\n",
       "5   103  11.0  12.0\n",
       "6   104  13.0  14.0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "success, data = tl.read(\n",
    "    c1=c1,\n",
    "    c2=c2,\n",
    "    allow_multiple=True,\n",
    "    return_as='df'\n",
    ")\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl.get_read_length(data=data, data_type='df')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Output as `dict` format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'time': [100, 101, 101, 102, 102, 103, 104],\n",
       " 'l1': [1.0, 3.0, 7.0, 5.0, 9.0, 11.0, 13.0],\n",
       " 'l2': [2.0, 4.0, 8.0, 6.0, 10.0, 12.0, 14.0]}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "success, data = tl.read(\n",
    "    c1=c1,\n",
    "    c2=c2,\n",
    "    allow_multiple=True,\n",
    "    return_as='dict'\n",
    ")\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl.get_read_length(data=data, data_type='dict')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Output as `sets-list` format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[({'iscrypto': 'yes',\n",
       "   'tl': 'test1',\n",
       "   'c1': 'crypto',\n",
       "   'c2': 'btc',\n",
       "   'timeframe': 'raw'},\n",
       "  [[100.0, 1.0, 2.0], [101.0, 3.0, 4.0], [102.0, 5.0, 6.0]]),\n",
       " ({'iscrypto': 'yes',\n",
       "   'tl': 'test1',\n",
       "   'c1': 'crypto',\n",
       "   'c2': 'eth',\n",
       "   'timeframe': 'raw'},\n",
       "  [[101.0, 7.0, 8.0],\n",
       "   [102.0, 9.0, 10.0],\n",
       "   [103.0, 11.0, 12.0],\n",
       "   [104.0, 13.0, 14.0]])]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "success, data = tl.read(\n",
    "    c1=c1,\n",
    "    c2=c2,\n",
    "    allow_multiple=True,\n",
    "    return_as='sets-list'\n",
    ")\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read_length returns the length of first set when reading as sets\n",
    "tl.get_read_length(data=data, data_type='sets-list')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Output as `sets-df` format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[({'iscrypto': 'yes',\n",
       "   'tl': 'test1',\n",
       "   'c1': 'crypto',\n",
       "   'c2': 'btc',\n",
       "   'timeframe': 'raw'},\n",
       "     time   l1   l2\n",
       "  0   100  1.0  2.0\n",
       "  1   101  3.0  4.0\n",
       "  2   102  5.0  6.0),\n",
       " ({'iscrypto': 'yes',\n",
       "   'tl': 'test1',\n",
       "   'c1': 'crypto',\n",
       "   'c2': 'eth',\n",
       "   'timeframe': 'raw'},\n",
       "     time    l1    l2\n",
       "  0   101   7.0   8.0\n",
       "  1   102   9.0  10.0\n",
       "  2   103  11.0  12.0\n",
       "  3   104  13.0  14.0)]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "success, data = tl.read(\n",
    "    c1=c1,\n",
    "    c2=c2,\n",
    "    allow_multiple=True,\n",
    "    return_as='sets-df'\n",
    ")\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read_length returns the length of first set when reading as sets\n",
    "tl.get_read_length(data=data, data_type='sets-df')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Output as `sets-dict` format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[({'iscrypto': 'yes',\n",
       "   'tl': 'test1',\n",
       "   'c1': 'crypto',\n",
       "   'c2': 'btc',\n",
       "   'timeframe': 'raw'},\n",
       "  {'time': [100, 101, 102], 'l1': [1.0, 3.0, 5.0], 'l2': [2.0, 4.0, 6.0]}),\n",
       " ({'iscrypto': 'yes',\n",
       "   'tl': 'test1',\n",
       "   'c1': 'crypto',\n",
       "   'c2': 'eth',\n",
       "   'timeframe': 'raw'},\n",
       "  {'time': [101, 102, 103, 104],\n",
       "   'l1': [7.0, 9.0, 11.0, 13.0],\n",
       "   'l2': [8.0, 10.0, 12.0, 14.0]})]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "success, data = tl.read(\n",
    "    c1=c1,\n",
    "    c2=c2,\n",
    "    allow_multiple=True,\n",
    "    return_as='sets-dict'\n",
    ")\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read_length returns the length of first set when reading as sets\n",
    "tl.get_read_length(data=data, data_type='sets-dict')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Output as `raw` format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'crypto_btc': {'count': 2,\n",
       "  'labels': {'iscrypto': 'yes',\n",
       "   'tl': 'test1',\n",
       "   'c1': 'crypto',\n",
       "   'c2': 'btc',\n",
       "   'timeframe': 'raw'},\n",
       "  'lines': {'l1': [(100000, 1.0), (101000, 3.0), (102000, 5.0)],\n",
       "   'l2': [(100000, 2.0), (101000, 4.0), (102000, 6.0)]}},\n",
       " 'crypto_eth': {'count': 2,\n",
       "  'labels': {'iscrypto': 'yes',\n",
       "   'tl': 'test1',\n",
       "   'c1': 'crypto',\n",
       "   'c2': 'eth',\n",
       "   'timeframe': 'raw'},\n",
       "  'lines': {'l1': [(101000, 7.0),\n",
       "    (102000, 9.0),\n",
       "    (103000, 11.0),\n",
       "    (104000, 13.0)],\n",
       "   'l2': [(101000, 8.0), (102000, 10.0), (103000, 12.0), (104000, 14.0)]}}}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "success, data = tl.read(\n",
    "    c1=c1,\n",
    "    c2=c2,\n",
    "    allow_multiple=True,\n",
    "    return_as='raw'\n",
    ")\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# due to efficiency, read_length returns the length of first set when reading as raw\n",
    "tl.get_read_length(data=data, data_type='raw')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10 (default, Sep 28 2021, 16:10:42) \n[GCC 9.3.0]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "2d702cda0469abe790a77633ff0c44a34d021c77dc3d2015e1d2d01a502c51d1"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}