span-batch-data-preparation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Span Batch Analysis - Data Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Before start, Fill in [secret.yml](span-batch-analysis/secret.yaml). Refer to [secret_example.yml](span-batch-analysis/secret_example.yaml). This is for specifying RPC endpoints.\n",
    "\n",
    "Also check config located at [config](span-batch-analysis/config/)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Build binaries."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "cd cmd/batch_decoder && go build . && cd ../../\n",
    "cd cmd/span_batch_tester && go build . && cd ../../"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "%env BATCH_DECODER=cmd/batch_decoder/batch_decoder\n",
    "%env SPAN_BATCH_TESTER=cmd/span_batch_tester/span_batch_tester"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Clone superchain config."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "git clone https://github.com/ethereum-optimism/superchain-registry\n",
    "cd superchain-registry && git checkout 5d3ca1966435827ac8c4216bddcb541ec64c1d6c && cd --"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Select your chain."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "%env L1_CHAIN_NAME=mainnet\n",
    "%env L2_CHAIN_NAME=op\n",
    "\n",
    "# available networks\n",
    "#      L1 - L2 list\n",
    "# mainnet - base, op, pgn, zora\n",
    "# goerli  - base, op, zora\n",
    "# sepolia - op, pgn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "l1_chain_name = os.environ[\"L1_CHAIN_NAME\"]\n",
    "l2_chain_name = os.environ[\"L2_CHAIN_NAME\"]\n",
    "\n",
    "start_number = !yq eval \".start_l1_number\" \"span-batch-analysis/config/{l1_chain_name}/{l2_chain_name}.yaml\"\n",
    "start_number_value = int(start_number[0])\n",
    "end_number = !yq eval \".end_l1_number\" \"span-batch-analysis/config/{l1_chain_name}/{l2_chain_name}.yaml\"\n",
    "end_number_value = int(end_number[0])\n",
    "batch_tx_sender = !yq eval \".batch_tx_sender\" \"span-batch-analysis/config/{l1_chain_name}/{l2_chain_name}.yaml\"\n",
    "batch_tx_sender_value = batch_tx_sender[0]\n",
    "l2_genesis_timestamp = !yq eval \".genesis.l2_time\" \"superchain-registry/superchain/configs/{l1_chain_name}/{l2_chain_name}.yaml\"\n",
    "l2_genesis_timestamp_value = int(l2_genesis_timestamp[0])\n",
    "l2_chain_id = !yq eval \".chain_id\" \"superchain-registry/superchain/configs/{l1_chain_name}/{l2_chain_name}.yaml\"\n",
    "l2_chain_id_value = int(l2_chain_id[0])\n",
    "inbox_addr = !yq eval \".batch_inbox_addr\" \"superchain-registry/superchain/configs/{l1_chain_name}/{l2_chain_name}.yaml\"\n",
    "inbox_addr_value = inbox_addr[0]\n",
    "\n",
    "acc_start_number = !yq eval \".acc_l2_start_number\" \"span-batch-analysis/config/{l1_chain_name}/{l2_chain_name}.yaml\"\n",
    "acc_start_number_value = int(acc_start_number[0])\n",
    "acc_end_number = !yq eval \".acc_l2_end_number\" \"span-batch-analysis/config/{l1_chain_name}/{l2_chain_name}.yaml\"\n",
    "acc_end_number_value = int(acc_end_number[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read analyis config."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%env L1_START=$start_number_value\n",
    "%env L1_END=$end_number_value\n",
    "%env BATCH_TX_SENDER=$batch_tx_sender_value\n",
    "%env L2_GENESIS_TIMESTAMP=$l2_genesis_timestamp_value\n",
    "%env L2_CHAIN_ID=$l2_chain_id_value\n",
    "%env INBOX=$inbox_addr_value\n",
    "\n",
    "%env ACC_L2_START=$acc_start_number_value\n",
    "%env ACC_L2_END=$acc_end_number_value\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get L1 RPC from secret."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "key_name = f\"L1_{l1_chain_name}_RPC\".upper()\n",
    "L1_RPC = !yq eval \".{key_name}\" span-batch-analysis/secret.yaml\n",
    "L1_RPC_ENDPOINT = L1_RPC[0]\n",
    "\n",
    "key_name = f\"L2_{l2_chain_name}_{l1_chain_name}_RPC\".upper()\n",
    "L2_RPC = !yq eval \".{key_name}\" span-batch-analysis/secret.yaml\n",
    "L2_RPC_ENDPOINT = L2_RPC[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "%env L1_RPC=$L1_RPC_ENDPOINT\n",
    "%env L2_RPC=$L2_RPC_ENDPOINT"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Collection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "All data will be stored at `./artifacts/{L1_CHAIN_NAME}/{L2_CHAIN_NAME}`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "l1_chain_name = os.environ[\"L1_CHAIN_NAME\"]\n",
    "l2_chain_name = os.environ[\"L2_CHAIN_NAME\"]\n",
    "\n",
    "%env DB_PREFIX=./artifacts/$l1_chain_name/$l2_chain_name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "batch decoder `fetch`: Read channel frames from L1 by reading batch transactions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!$BATCH_DECODER fetch --l1=$L1_RPC --start=$L1_START --end=$L1_END --inbox=$INBOX --out=$DB_PREFIX/transactions_cache --sender=$BATCH_TX_SENDER --concurrent-requests=25 | tail -10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "batch decoder `reassemble`: Reassemble channel frames to channels. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!$BATCH_DECODER reassemble --inbox=$INBOX --in=$DB_PREFIX/transactions_cache --out=$DB_PREFIX/channel_cache | tail -10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "span batch tester `convert`: Convert channels with singular batches to span batches. Try multiple tx encoding types."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!TX_ENCODE_TYPE=0  && $SPAN_BATCH_TESTER convert --in=$DB_PREFIX/channel_cache --out=$DB_PREFIX/span_batch_cache_v$TX_ENCODE_TYPE --l2=$L2_RPC --genesis-timestamp=$L2_GENESIS_TIMESTAMP --chain-id=$L2_CHAIN_ID --tx-type=$TX_ENCODE_TYPE | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!TX_ENCODE_TYPE=1  && $SPAN_BATCH_TESTER convert --in=$DB_PREFIX/channel_cache --out=$DB_PREFIX/span_batch_cache_v$TX_ENCODE_TYPE --l2=$L2_RPC --genesis-timestamp=$L2_GENESIS_TIMESTAMP --chain-id=$L2_CHAIN_ID --tx-type=$TX_ENCODE_TYPE | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!TX_ENCODE_TYPE=2  && $SPAN_BATCH_TESTER convert --in=$DB_PREFIX/channel_cache --out=$DB_PREFIX/span_batch_cache_v$TX_ENCODE_TYPE --l2=$L2_RPC --genesis-timestamp=$L2_GENESIS_TIMESTAMP --chain-id=$L2_CHAIN_ID --tx-type=$TX_ENCODE_TYPE | tail -10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "span batch tester `analyze`: Analyze channels with singluar batches by comparing with corresponding span batches."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!TX_ENCODE_TYPE=0 && $SPAN_BATCH_TESTER analyze --in-channel=$DB_PREFIX/channel_cache --in-span-batch=$DB_PREFIX/span_batch_cache_v$TX_ENCODE_TYPE --out=$DB_PREFIX/result_v$TX_ENCODE_TYPE --chain-id=$L2_CHAIN_ID --tx-type=$TX_ENCODE_TYPE | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!TX_ENCODE_TYPE=1 && $SPAN_BATCH_TESTER analyze --in-channel=$DB_PREFIX/channel_cache --in-span-batch=$DB_PREFIX/span_batch_cache_v$TX_ENCODE_TYPE --out=$DB_PREFIX/result_v$TX_ENCODE_TYPE --chain-id=$L2_CHAIN_ID --tx-type=$TX_ENCODE_TYPE | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!TX_ENCODE_TYPE=2 && $SPAN_BATCH_TESTER analyze --in-channel=$DB_PREFIX/channel_cache --in-span-batch=$DB_PREFIX/span_batch_cache_v$TX_ENCODE_TYPE --out=$DB_PREFIX/result_v$TX_ENCODE_TYPE --chain-id=$L2_CHAIN_ID --tx-type=$TX_ENCODE_TYPE | tail -10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "span batch tester `fetch`: Fetch consecutive singular batches."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!$SPAN_BATCH_TESTER fetch --l2 $L2_RPC --start=$ACC_L2_START --end=$ACC_L2_END --concurrent-requests=100 --out=$DB_PREFIX/singular_batches_cache | tail -10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "span batch tester `merge`: Merge singular batches:\n",
    "\n",
    "This command will analyze `start - end` number of batches, Comparing uncompressed size of below two batches:\n",
    "- For `i` in range  `0` to `start - end - 1`\n",
    "  - singular batch from number `start` to `start + i`, inclusive.\n",
    "  - span batch from number `start` to `start + i`, inclusive.\n",
    "\n",
    "Upper command will make a result json file which has name of the form `{start_number}_{end_number}.json`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!$SPAN_BATCH_TESTER merge --start=$ACC_L2_START --end=$ACC_L2_END --l2 $L2_RPC --genesis-timestamp=$L2_GENESIS_TIMESTAMP --in=$DB_PREFIX/singular_batches_cache --out=$DB_PREFIX/merge_result | tail -10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "span batch tester `format`: Compare different tx encodings for span batch.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!$SPAN_BATCH_TESTER format --permutation 0123465 --in-span-batch=$DB_PREFIX/span_batch_cache_v2 --out=$DB_PREFIX/format_result --chain-id=$L2_CHAIN_ID | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!$SPAN_BATCH_TESTER format --permutation 0134625 --in-span-batch=$DB_PREFIX/span_batch_cache_v2 --out=$DB_PREFIX/format_result --chain-id=$L2_CHAIN_ID | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!$SPAN_BATCH_TESTER format --permutation 3041265 --in-span-batch=$DB_PREFIX/span_batch_cache_v2 --out=$DB_PREFIX/format_result --chain-id=$L2_CHAIN_ID | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!$SPAN_BATCH_TESTER format --permutation 0125634 --in-span-batch=$DB_PREFIX/span_batch_cache_v2 --out=$DB_PREFIX/format_result --chain-id=$L2_CHAIN_ID | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!$SPAN_BATCH_TESTER format --permutation 0324165 --in-span-batch=$DB_PREFIX/span_batch_cache_v2 --out=$DB_PREFIX/format_result --chain-id=$L2_CHAIN_ID | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!$SPAN_BATCH_TESTER format --permutation 3041256 --in-span-batch=$DB_PREFIX/span_batch_cache_v2 --out=$DB_PREFIX/format_result --chain-id=$L2_CHAIN_ID | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!$SPAN_BATCH_TESTER format --permutation 3401256 --in-span-batch=$DB_PREFIX/span_batch_cache_v2 --out=$DB_PREFIX/format_result --chain-id=$L2_CHAIN_ID | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!$SPAN_BATCH_TESTER format --permutation 6051234 --in-span-batch=$DB_PREFIX/span_batch_cache_v2 --out=$DB_PREFIX/format_result --chain-id=$L2_CHAIN_ID | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!$SPAN_BATCH_TESTER format --permutation 0132456 --in-span-batch=$DB_PREFIX/span_batch_cache_v2 --out=$DB_PREFIX/format_result --chain-id=$L2_CHAIN_ID | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!$SPAN_BATCH_TESTER format --permutation 0352614 --in-span-batch=$DB_PREFIX/span_batch_cache_v2 --out=$DB_PREFIX/format_result --chain-id=$L2_CHAIN_ID | tail -10"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}