notebooks/view_data_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('./../')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "from src.dataset.t5data import T5Data\n",
    "from src.io import read_jsonl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add the data to the appropriate path for the next cells to function."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TCA1D or SE Indirect 1D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset health:\n",
      "# input data: 8489\n",
      "# output data: 8489\n"
     ]
    }
   ],
   "source": [
    "dpath = Path(\"./../training_data/se_indirect_1d/test.jsonl\")\n",
    "tca1d = T5Data(dpath, task='tca1d')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input': '+92% ',\n",
       " 'output': 'property_value',\n",
       " 'input_type': 'text',\n",
       " 'augmentation_status': 'original'}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tca1d[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TCA2D or SE Indirect 2D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset health:\n",
      "# input data: 22839\n",
      "# output data: 22839\n"
     ]
    }
   ],
   "source": [
    "dpath = Path(\"./../training_data/se_indirect_2d/test.jsonl\")\n",
    "tca2d = T5Data(dpath, task='tca2d')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| 0                     | 1       | 2       | 3        |\n",
      "|-----------------------|---------|---------|----------|\n",
      "|                       | FY2021  | FY2022  | % change |\n",
      "| Scope 2 emissions     | 85,341  | 85,050  | -0.3%    |\n",
      "| Scope 1 emissions     | 27,716  | 28,500  | 2.8%     |\n",
      "| Scope 1 & 2 emissions | 113,057 | 113,550 | 0.4%     |\n"
     ]
    }
   ],
   "source": [
    "# let look at the input\n",
    "print(tca2d[167]['input'].replace(T5Data.newline_token, '\\n'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| 0        | 1              | 2              | 3              |\n",
      "|----------|----------------|----------------|----------------|\n",
      "| empty    | time_value     | time_value     | unit_value     |\n",
      "| property | property_value | property_value | property_value |\n",
      "| property | property_value | property_value | property_value |\n",
      "| property | property_value | property_value | property_value |\n"
     ]
    }
   ],
   "source": [
    "# let look at the output\n",
    "print(tca2d[167]['output'].replace(T5Data.newline_token, '\\n'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SE Direct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset health:\n",
      "# input data: 11682\n",
      "# output data: 11682\n"
     ]
    }
   ],
   "source": [
    "dpath = Path(\"./../training_data/se_direct/test.jsonl\")\n",
    "ud2sd = T5Data(dpath, task='ud2sd_table')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| 0                                                               | 1       | 2       | 3            |\n",
      "|-----------------------------------------------------------------|---------|---------|--------------|\n",
      "| Australia - Scope 1                                             | 2021    | 2020    | 2022         |\n",
      "| Emissions from fuel consumption (tCO$_{2}$e)                    | 17,957  | 14,574  | 19,561       |\n",
      "| Emissions from Killara Feedlot cattle (tCO$_{2}$e)              | 37,462  | -       | 44,826       |\n",
      "| Fuel consumption (GJ)                                           | 255,873 | 207,569 | 278,969      |\n",
      "| Australia - Scope 2                                             |         |         |              |\n",
      "| Electricity consumption from the grid (GJ)                      | 24,201  | 41,051  | 29,104       |\n",
      "| Total renewable electricity (MWh)                               | -       | -       | 8,084 (100%) |\n",
      "| Voluntary LGCs                                                  | -       | -       | 6,581        |\n",
      "| Electricity consumption from the grid (MWh)                     | 6,722   | 11,403  | 8,084        |\n",
      "| Mandatory LGCs 1                                                | -       | -       | 1,503        |\n",
      "| Emissions from purchased electricity (tCO$_{2}$e) 2             | 4,982   | 8,946   | 5,801        |\n",
      "| Total energy (Australia)                                        |         |         |              |\n",
      "| Total energy consumption (GJ)                                   | 280,074 |         | 308,073      |\n",
      "| China - Scope 2                                                 |         |         |              |\n",
      "| Electricity consumption from the grid (GJ)                      | 2,466   | -       | 2,580        |\n",
      "| Scope 2 GHG emissions from electricity consumption (tCO$_{2}$e) | 427     | -       | 385          |\n"
     ]
    }
   ],
   "source": [
    "# let look at the input\n",
    "print(ud2sd[2]['input'].replace(T5Data.newline_token, '\\n'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| property                                                           | property_value   | unit   | subject   | subject_value   |\n",
      "|--------------------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 1 : Emissions from fuel consumption (tCO$_{2}$e) | 17,957           |        |           |                 |\n",
      "| time                                                               | 2021             |        |           |                 |\n",
      "\n",
      "| property                                                           | property_value   | unit   | subject   | subject_value   |\n",
      "|--------------------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 1 : Emissions from fuel consumption (tCO$_{2}$e) | 14,574           |        |           |                 |\n",
      "| time                                                               | 2020             |        |           |                 |\n",
      "\n",
      "| property                                                           | property_value   | unit   | subject   | subject_value   |\n",
      "|--------------------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 1 : Emissions from fuel consumption (tCO$_{2}$e) | 19,561           |        |           |                 |\n",
      "| time                                                               | 2022             |        |           |                 |\n",
      "\n",
      "| property                                                                 | property_value   | unit   | subject   | subject_value   |\n",
      "|--------------------------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 1 : Emissions from Killara Feedlot cattle (tCO$_{2}$e) | 37,462           |        |           |                 |\n",
      "| time                                                                     | 2021             |        |           |                 |\n",
      "\n",
      "| property                                                                 | property_value   | unit   | subject   | subject_value   |\n",
      "|--------------------------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 1 : Emissions from Killara Feedlot cattle (tCO$_{2}$e) | -                |        |           |                 |\n",
      "| time                                                                     | 2020             |        |           |                 |\n",
      "\n",
      "| property                                                                 | property_value   | unit   | subject   | subject_value   |\n",
      "|--------------------------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 1 : Emissions from Killara Feedlot cattle (tCO$_{2}$e) | 44,826           |        |           |                 |\n",
      "| time                                                                     | 2022             |        |           |                 |\n",
      "\n",
      "| property                                    | property_value   | unit   | subject   | subject_value   |\n",
      "|---------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 1 : Fuel consumption (GJ) | 255,873          |        |           |                 |\n",
      "| time                                        | 2021             |        |           |                 |\n",
      "\n",
      "| property                                    | property_value   | unit   | subject   | subject_value   |\n",
      "|---------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 1 : Fuel consumption (GJ) | 207,569          |        |           |                 |\n",
      "| time                                        | 2020             |        |           |                 |\n",
      "\n",
      "| property                                    | property_value   | unit   | subject   | subject_value   |\n",
      "|---------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 1 : Fuel consumption (GJ) | 278,969          |        |           |                 |\n",
      "| time                                        | 2022             |        |           |                 |\n",
      "\n",
      "| property                                                         | property_value   | unit   | subject   | subject_value   |\n",
      "|------------------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 2 : Electricity consumption from the grid (GJ) | 24,201           |        |           |                 |\n",
      "| time                                                             | 2021             |        |           |                 |\n",
      "\n",
      "| property                                                         | property_value   | unit   | subject   | subject_value   |\n",
      "|------------------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 2 : Electricity consumption from the grid (GJ) | 41,051           |        |           |                 |\n",
      "| time                                                             | 2020             |        |           |                 |\n",
      "\n",
      "| property                                                         | property_value   | unit   | subject   | subject_value   |\n",
      "|------------------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 2 : Electricity consumption from the grid (GJ) | 29,104           |        |           |                 |\n",
      "| time                                                             | 2022             |        |           |                 |\n",
      "\n",
      "| property                                                | property_value   | unit   | subject   | subject_value   |\n",
      "|---------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 2 : Total renewable electricity (MWh) | -                |        |           |                 |\n",
      "| time                                                    | 2021             |        |           |                 |\n",
      "\n",
      "| property                                                | property_value   | unit   | subject   | subject_value   |\n",
      "|---------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 2 : Total renewable electricity (MWh) | -                |        |           |                 |\n",
      "| time                                                    | 2020             |        |           |                 |\n",
      "\n",
      "| property                                                | property_value   | unit   | subject   | subject_value   |\n",
      "|---------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 2 : Total renewable electricity (MWh) | 8,084 (100%)     |        |           |                 |\n",
      "| time                                                    | 2022             |        |           |                 |\n",
      "\n",
      "| property                             | property_value   | unit   | subject   | subject_value   |\n",
      "|--------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 2 : Voluntary LGCs | -                |        |           |                 |\n",
      "| time                                 | 2021             |        |           |                 |\n",
      "\n",
      "| property                             | property_value   | unit   | subject   | subject_value   |\n",
      "|--------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 2 : Voluntary LGCs | -                |        |           |                 |\n",
      "| time                                 | 2020             |        |           |                 |\n",
      "\n",
      "| property                             | property_value   | unit   | subject   | subject_value   |\n",
      "|--------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 2 : Voluntary LGCs | 6,581            |        |           |                 |\n",
      "| time                                 | 2022             |        |           |                 |\n",
      "\n",
      "| property                                                          | property_value   | unit   | subject   | subject_value   |\n",
      "|-------------------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 2 : Electricity consumption from the grid (MWh) | 6,722            |        |           |                 |\n",
      "| time                                                              | 2021             |        |           |                 |\n",
      "\n",
      "| property                                                          | property_value   | unit   | subject   | subject_value   |\n",
      "|-------------------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 2 : Electricity consumption from the grid (MWh) | 11,403           |        |           |                 |\n",
      "| time                                                              | 2020             |        |           |                 |\n",
      "\n",
      "| property                                                          | property_value   | unit   | subject   | subject_value   |\n",
      "|-------------------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 2 : Electricity consumption from the grid (MWh) | 8,084            |        |           |                 |\n",
      "| time                                                              | 2022             |        |           |                 |\n",
      "\n",
      "| property                               | property_value   | unit   | subject   | subject_value   |\n",
      "|----------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 2 : Mandatory LGCs 1 | -                |        |           |                 |\n",
      "| time                                   | 2021             |        |           |                 |\n",
      "\n",
      "| property                               | property_value   | unit   | subject   | subject_value   |\n",
      "|----------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 2 : Mandatory LGCs 1 | -                |        |           |                 |\n",
      "| time                                   | 2020             |        |           |                 |\n",
      "\n",
      "| property                               | property_value   | unit   | subject   | subject_value   |\n",
      "|----------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 2 : Mandatory LGCs 1 | 1,503            |        |           |                 |\n",
      "| time                                   | 2022             |        |           |                 |\n",
      "\n",
      "| property                                                                  | property_value   | unit   | subject   | subject_value   |\n",
      "|---------------------------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 2 : Emissions from purchased electricity (tCO$_{2}$e) 2 | 4,982            |        |           |                 |\n",
      "| time                                                                      | 2021             |        |           |                 |\n",
      "\n",
      "| property                                                                  | property_value   | unit   | subject   | subject_value   |\n",
      "|---------------------------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 2 : Emissions from purchased electricity (tCO$_{2}$e) 2 | 8,946            |        |           |                 |\n",
      "| time                                                                      | 2020             |        |           |                 |\n",
      "\n",
      "| property                                                                  | property_value   | unit   | subject   | subject_value   |\n",
      "|---------------------------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Australia - Scope 2 : Emissions from purchased electricity (tCO$_{2}$e) 2 | 5,801            |        |           |                 |\n",
      "| time                                                                      | 2022             |        |           |                 |\n",
      "\n",
      "| property                                                 | property_value   | unit   | subject   | subject_value   |\n",
      "|----------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Total energy (Australia) : Total energy consumption (GJ) | 280,074          |        |           |                 |\n",
      "| time                                                     | 2021             |        |           |                 |\n",
      "\n",
      "| property                                                 | property_value   | unit   | subject   | subject_value   |\n",
      "|----------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| Total energy (Australia) : Total energy consumption (GJ) | 308,073          |        |           |                 |\n",
      "| time                                                     | 2022             |        |           |                 |\n",
      "\n",
      "| property                                                     | property_value   | unit   | subject   | subject_value   |\n",
      "|--------------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| China - Scope 2 : Electricity consumption from the grid (GJ) | 2,466            |        |           |                 |\n",
      "| time                                                         | 2021             |        |           |                 |\n",
      "\n",
      "| property                                                     | property_value   | unit   | subject   | subject_value   |\n",
      "|--------------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| China - Scope 2 : Electricity consumption from the grid (GJ) | -                |        |           |                 |\n",
      "| time                                                         | 2020             |        |           |                 |\n",
      "\n",
      "| property                                                     | property_value   | unit   | subject   | subject_value   |\n",
      "|--------------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| China - Scope 2 : Electricity consumption from the grid (GJ) | 2,580            |        |           |                 |\n",
      "| time                                                         | 2022             |        |           |                 |\n",
      "\n",
      "| property                                                                          |   property_value | unit   | subject   | subject_value   |\n",
      "|-----------------------------------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| China - Scope 2 : Scope 2 GHG emissions from electricity consumption (tCO$_{2}$e) |             427  |        |           |                 |\n",
      "| time                                                                              |            2021  |        |           |                 |\n",
      "\n",
      "| property                                                                          | property_value   | unit   | subject   | subject_value   |\n",
      "|-----------------------------------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| China - Scope 2 : Scope 2 GHG emissions from electricity consumption (tCO$_{2}$e) | -                |        |           |                 |\n",
      "| time                                                                              | 2020             |        |           |                 |\n",
      "\n",
      "| property                                                                          |   property_value | unit   | subject   | subject_value   |\n",
      "|-----------------------------------------------------------------------------------|------------------|--------|-----------|-----------------|\n",
      "| China - Scope 2 : Scope 2 GHG emissions from electricity consumption (tCO$_{2}$e) |             385  |        |           |                 |\n",
      "| time                                                                              |            2022  |        |           |                 |\n"
     ]
    }
   ],
   "source": [
    "# let look at the output\n",
    "print(ud2sd[2]['output'].replace(T5Data.newline_token, '\\n').replace(T5Data.sep_token, '\\n\\n'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}