scenario3_backtest.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### PPO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "from stable_baselines3 import PPO\n",
    "\n",
    "import torch.nn as nn\n",
    "import torch\n",
    "from cryptoportfolio.rlagent.rlagent_ppo import RLAgent as ppo_agent\n",
    "from cryptoportfolio.rlagent.network import CustomCNN_PPO\n",
    "from cryptoportfolio.rlagent.network import CustomActorCriticPolicy \n",
    "import pprint\n",
    "import torch.nn as nn\n",
    "from stable_baselines3.common.buffers import RolloutBuffer\n",
    "\n",
    "from stable_baselines3.common.vec_env import DummyVecEnv\n",
    "from stable_baselines3.common.monitor import Monitor\n",
    "from cryptoportfolio.tools.eval_mod import custom_eval_callback_scenario3\n",
    "from cryptoportfolio.tools.eval_mod import custom_eval_callback_testset\n",
    "\n",
    "features_list = {\"None\": [],\n",
    "        \"RSI\": [\"RSI\"],\n",
    "        \"mcd\": [\"mcd\", \"mcd_signal\"],\n",
    "        \"SMA\": [\"SMA_50\", \"SMA_200\"],\n",
    "        \"RSI_mcd\": [\"RSI\", \"mcd\", \"mcd_signal\"],\n",
    "        \"RSI_SMA\": [\"RSI\", \"SMA_50\", \"SMA_200\"],\n",
    "        \"OSZ\": [\"stoch_oszillator\", \"stoch_oszillator_signal\"],\n",
    "        \"RSI_OSZ\": [\"RSI\", \"stoch_oszillator\", \"stoch_oszillator_signal\"],\n",
    "        \"Bollinger\": [\"Bollinger_middle\" ,\"Bollinger_low\" ,\"Bollinger_high\"],\n",
    "        \"Bollinger_mcd\": [\"mcd\", \"mcd_signal\", \"Bollinger_middle\" ,\"Bollinger_low\" ,\"Bollinger_high\"],\n",
    "        \"RSI_SMA_Bollinger\": [\"RSI\", \"SMA_50\", \"SMA_200\", \"Bollinger_middle\" ,\"Bollinger_low\" ,\"Bollinger_high\"],\n",
    "        \"mcd_OSZ\": [\"mcd\", \"mcd_signal\", \"stoch_oszillator\", \"stoch_oszillator_signal\"],\n",
    "        \"Everything\": [\"RSI\", \"SMA_50\", \"SMA_200\", \"Bollinger_middle\" ,\"Bollinger_low\" ,\"Bollinger_high\", \"mcd\", \"mcd_signal\", \"stoch_oszillator\", \"stoch_oszillator_signal\"]\n",
    "    }\n",
    "activation_fn = {\"tanh\": nn.Tanh, \"relu\": nn.ReLU}\n",
    "\n",
    "def backtest_ppo(hyperparams, total_iterations, eval_freq, load_model=False, scenario=None):\n",
    "    timer = int(time.time())\n",
    "    \n",
    "    pp = pprint.PrettyPrinter(indent=0)\n",
    "    pp.pprint(hyperparams)\n",
    "\n",
    "    \"\"\" First iteration \"\"\"\n",
    "    agent = ppo_agent(lookback_window_size=hyperparams[\"window_size\"],\n",
    "                      features=hyperparams[\"features\"],\n",
    "                      batch_size=hyperparams[\"batch_size\"])\n",
    "    agent.generate_portfolio(year=\"2018\", synthetic=False, split=\"train\")\n",
    "    n_steps = int(agent._crash_length - agent._lookback_window_size)\n",
    "    hyperparams.update({\"n_steps\": n_steps})\n",
    "    hyperparams.update({\"env\": agent})\n",
    "    hyperparams[\"policy_kwargs\"].update(\n",
    "        {\"features_extractor_class\": CustomCNN_PPO,\n",
    "        \"features_extractor_kwargs\": dict(features_dim=13, agent_env=agent)}        \n",
    "    )\n",
    "    \n",
    "    if load_model is not None:\n",
    "        # Wrap the env\n",
    "        env = Monitor(agent)\n",
    "        env = DummyVecEnv([lambda: env])\n",
    "        _ = env.reset()\n",
    "\n",
    "        # Load the model and overwrite env\n",
    "        model = PPO.load(f\"models/ppo/{load_model}\")\n",
    "        model.env = env\n",
    "        print(\"PPO policy:\", model.policy)\n",
    "        \n",
    "    else:\n",
    "        logdir = f\"logs/ppo/{scenario}\"        \n",
    "        model = PPO(\n",
    "            policy = hyperparams[\"policy\"],\n",
    "            env = hyperparams[\"env\"],\n",
    "            learning_rate = hyperparams[\"learning_rate\"],\n",
    "            n_steps = hyperparams[\"n_steps\"],\n",
    "            batch_size = hyperparams[\"batch_size\"],\n",
    "            n_epochs = hyperparams[\"n_epochs\"],\n",
    "            gamma = hyperparams[\"gamma\"],\n",
    "            gae_lambda = hyperparams[\"gae_lambda\"],\n",
    "            clip_range = hyperparams[\"clip_range\"],\n",
    "            ent_coef = hyperparams[\"ent_coef\"],\n",
    "            vf_coef = hyperparams[\"vf_coef\"],\n",
    "            max_grad_norm = hyperparams[\"max_grad_norm\"],\n",
    "            tensorboard_log = logdir,\n",
    "            policy_kwargs = hyperparams[\"policy_kwargs\"],\n",
    "            verbose = 1,\n",
    "            seed = hyperparams[\"seed\"],\n",
    "        )\n",
    "\n",
    "    total_timesteps = total_iterations * n_steps\n",
    "\n",
    "    # Creating the custom EvalCallback\n",
    "    #eval_callback = custom_eval_callback_scenario3(hyperparams, total_timesteps, \"PPO\", eval_freq)\n",
    "    eval_callback = custom_eval_callback_testset(hyperparams, total_timesteps, \"PPO\", eval_freq)\n",
    "    model.learn(total_timesteps, tb_log_name=f\"{timer}\", callback=eval_callback)\n",
    "\n",
    "    \"\"\" Second iteration \"\"\"\n",
    "    agent = ppo_agent(lookback_window_size=hyperparams[\"window_size\"],\n",
    "                      features=hyperparams[\"features\"],\n",
    "                      batch_size=hyperparams[\"batch_size\"])\n",
    "    agent.generate_portfolio(year=\"2021\", synthetic=False, split=\"train\")\n",
    "    n_steps = int(agent._crash_length - agent._lookback_window_size)\n",
    "    \n",
    "    # Overwriting the model parameters with the new ones\n",
    "    model.n_steps = n_steps\n",
    "    model.env.envs[0].env = agent\n",
    "    \n",
    "    model.rollout_buffer = RolloutBuffer(\n",
    "            model.n_steps,\n",
    "            model.observation_space,\n",
    "            model.action_space,\n",
    "            device=model.device,\n",
    "            gamma=model.gamma,\n",
    "            gae_lambda=model.gae_lambda,\n",
    "            n_envs=model.n_envs,\n",
    "    )\n",
    "    total_timesteps = total_iterations * n_steps\n",
    "    \n",
    "    # Creating the custom EvalCallback\n",
    "    #eval_callback = custom_eval_callback_scenario3(hyperparams, total_timesteps, \"PPO\", eval_freq)\n",
    "    eval_callback = custom_eval_callback_testset(hyperparams, total_timesteps, \"PPO\", eval_freq)\n",
    "    model.learn(total_timesteps, tb_log_name=f\"{timer}\", callback=eval_callback)\n",
    "    \n",
    "    \"\"\" Third Iteration \"\"\"\n",
    "    agent = ppo_agent(lookback_window_size=hyperparams[\"window_size\"],\n",
    "                      features=hyperparams[\"features\"],\n",
    "                      batch_size=hyperparams[\"batch_size\"])\n",
    "    agent.generate_portfolio(year=\"2018\", synthetic=True, split=\"train\")\n",
    "    n_steps = int(agent._crash_length - agent._lookback_window_size)\n",
    "    \n",
    "    # Overwriting the model parameters with the new ones\n",
    "    model.n_steps = n_steps\n",
    "    model.env.envs[0].env = agent\n",
    "    \n",
    "    model.rollout_buffer = RolloutBuffer(\n",
    "            model.n_steps,\n",
    "            model.observation_space,\n",
    "            model.action_space,\n",
    "            device=model.device,\n",
    "            gamma=model.gamma,\n",
    "            gae_lambda=model.gae_lambda,\n",
    "            n_envs=model.n_envs,\n",
    "    )\n",
    "    total_timesteps = total_iterations * n_steps\n",
    "    \n",
    "    # Creating the custom EvalCallback\n",
    "    #eval_callback = custom_eval_callback_scenario3(hyperparams, total_timesteps, \"PPO\", eval_freq)\n",
    "    eval_callback = custom_eval_callback_testset(hyperparams, total_timesteps, \"PPO\", eval_freq)\n",
    "    model.learn(total_timesteps, tb_log_name=f\"{timer}\", callback=eval_callback)\n",
    "    \n",
    "    \"\"\" Fourth Iteration \"\"\"\n",
    "    agent = ppo_agent(lookback_window_size=hyperparams[\"window_size\"],\n",
    "                      features=hyperparams[\"features\"],\n",
    "                      batch_size=hyperparams[\"batch_size\"])\n",
    "    agent.generate_portfolio(year=\"2021\", synthetic=True, split=\"train\")\n",
    "    n_steps = int(agent._crash_length - agent._lookback_window_size)\n",
    "    \n",
    "    # Overwriting the model parameters with the new ones\n",
    "    model.n_steps = n_steps\n",
    "    model.env.envs[0].env = agent\n",
    "    \n",
    "    model.rollout_buffer = RolloutBuffer(\n",
    "            model.n_steps,\n",
    "            model.observation_space,\n",
    "            model.action_space,\n",
    "            device=model.device,\n",
    "            gamma=model.gamma,\n",
    "            gae_lambda=model.gae_lambda,\n",
    "            n_envs=model.n_envs,\n",
    "    )\n",
    "    total_timesteps = total_iterations * n_steps\n",
    "    \n",
    "    # Creating the custom EvalCallback\n",
    "    #eval_callback = custom_eval_callback_scenario3(hyperparams, total_timesteps, \"PPO\", eval_freq)\n",
    "    eval_callback = custom_eval_callback_testset(hyperparams, total_timesteps, \"PPO\", eval_freq)\n",
    "    model.learn(total_timesteps, tb_log_name=f\"{timer}\", callback=eval_callback)\n",
    "    \n",
    "    \"\"\" Backtest \"\"\"\n",
    "    test_agent = ppo_agent(lookback_window_size=hyperparams[\"window_size\"],\n",
    "                      features=hyperparams[\"features\"],\n",
    "                      batch_size=hyperparams[\"batch_size\"])\n",
    "    test_agent.generate_portfolio(year=\"2022\", synthetic=False, split=\"whole\")\n",
    "\n",
    "    done = False\n",
    "    obs = test_agent.reset()\n",
    "    while done is False:\n",
    "        w, _ = model.predict(torch.from_numpy(obs).float(), deterministic=True)\n",
    "        obs, reward, done, info = test_agent.step(w)\n",
    "    test_agent.render()\n",
    "    \n",
    "    return test_agent, model\n",
    "\n",
    "# zeros-nosoft-cnn-mlp-cheating-new-lr #114 #best so far with 0.371 # 1686647972\n",
    "hyperparams = {\n",
    "        \"policy\": CustomActorCriticPolicy, \n",
    "        \"clip_range\": 0.1,\n",
    "        \"ent_coef\" : 0.0028646621529157634,\n",
    "        \"features\" : [\"close\", \"low\", \"high\"] + features_list[\"RSI_SMA_Bollinger\"],\n",
    "        \"gae_lambda\": 0.99,\n",
    "        \"gamma\": 0.95,\n",
    "        \"learning_rate\" : 0.0001735302876108352,\n",
    "        \"max_grad_norm\" : 5.0,\n",
    "        \"n_epochs\" : 25,\n",
    "        \"seed\" : 1028,\n",
    "        \"vf_coef\" : 0.004473747041794429,\n",
    "        \"window_size\" : 150,\n",
    "        \"batch_size\" : 15,\n",
    "        \"policy_kwargs\": dict(\n",
    "            layer_size=256,\n",
    "            n_layers=3,\n",
    "            activation_fn=activation_fn[\"tanh\"],\n",
    "            ortho_init=False,\n",
    "        )}\n",
    "\n",
    "#total_iterations = 30; eval_freq = 10\n",
    "total_iterations = 1; eval_freq = 1\n",
    "scenario = f\"scenario3_nosoft-cnn-mlp_it{total_iterations}\"; load_model = None#\"scenario3_nosoft-cnn-mlp_it50\"\n",
    "test_agent, model = backtest_ppo(hyperparams, total_iterations=total_iterations, eval_freq=eval_freq, load_model=load_model, scenario=scenario)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Saving the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(f\"models/ppo/{scenario}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Saving the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cryptoportfolio.tools.performance_measures import max_drawdown, sharpe_ratio\n",
    "\n",
    "# Save the results\n",
    "data = open(\"logs/baselines_performance_metrics.csv\", \"a\")\n",
    "data.write(\"\\n\")\n",
    "data.write(test_agent._name + \"_\" + scenario); data.write(\",\")\n",
    "data.write(\"2022\"); data.write(\",\")\n",
    "data.write(str(test_agent._lookback_window_size)); data.write(\",\")\n",
    "data.write(str(max_drawdown(test_agent._portfolio_values))); data.write(\",\")\n",
    "data.write(str(test_agent._portfolio_values[-1])); data.write(\",\")\n",
    "data.write(str(sharpe_ratio(test_agent._rates_of_return))); data.write(\",\")\n",
    "data.write(str(test_agent._sum_of_transaction_costs))\n",
    "data.close()\n",
    "\n",
    "# Save the results\n",
    "data = open(\"logs/baselines_portfolio_values.csv\", \"a\")   \n",
    "data.write(\"\\n\")\n",
    "data.write(test_agent._name + \"_\" + scenario); data.write(\";\")\n",
    "data.write(\"2022\"); data.write(\";\")\n",
    "data.write(str(test_agent._lookback_window_size)); data.write(\";\")\n",
    "data.write(str(list(test_agent._portfolio_values)))\n",
    "data.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### DDPG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime, time\n",
    "import pandas as pd\n",
    "import gym\n",
    "import numpy as np\n",
    "from stable_baselines3 import PPO, DDPG\n",
    "import torch.nn as nn\n",
    "import torch\n",
    "from typing import Any, Dict\n",
    "\n",
    "from stable_baselines3 import DDPG\n",
    "from cryptoportfolio.rlagent.rlagent_ddpg import RLAgent as ddpg_agent\n",
    "from cryptoportfolio.rlagent.network import CustomCNN_DDPG\n",
    "import pprint\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from stable_baselines3.common.vec_env import DummyVecEnv\n",
    "from stable_baselines3.common.monitor import Monitor\n",
    "from cryptoportfolio.tools.eval_mod import custom_eval_callback_scenario3\n",
    "from cryptoportfolio.tools.eval_mod import custom_eval_callback_testset\n",
    "\n",
    "features_list = {\"None\": [],\n",
    "        \"RSI\": [\"RSI\"],\n",
    "        \"mcd\": [\"mcd\", \"mcd_signal\"],\n",
    "        \"SMA\": [\"SMA_50\", \"SMA_200\"],\n",
    "        \"RSI_mcd\": [\"RSI\", \"mcd\", \"mcd_signal\"],\n",
    "        \"RSI_SMA\": [\"RSI\", \"SMA_50\", \"SMA_200\"],\n",
    "        \"OSZ\": [\"stoch_oszillator\", \"stoch_oszillator_signal\"],\n",
    "        \"RSI_OSZ\": [\"RSI\", \"stoch_oszillator\", \"stoch_oszillator_signal\"],\n",
    "        \"Bollinger\": [\"Bollinger_middle\" ,\"Bollinger_low\" ,\"Bollinger_high\"],\n",
    "        \"Bollinger_mcd\": [\"mcd\", \"mcd_signal\", \"Bollinger_middle\" ,\"Bollinger_low\" ,\"Bollinger_high\"],\n",
    "        \"RSI_SMA_Bollinger\": [\"RSI\", \"SMA_50\", \"SMA_200\", \"Bollinger_middle\" ,\"Bollinger_low\" ,\"Bollinger_high\"],\n",
    "        \"mcd_OSZ\": [\"mcd\", \"mcd_signal\", \"stoch_oszillator\", \"stoch_oszillator_signal\"],\n",
    "        \"Everything\": [\"RSI\", \"SMA_50\", \"SMA_200\", \"Bollinger_middle\" ,\"Bollinger_low\" ,\"Bollinger_high\", \"mcd\", \"mcd_signal\", \"stoch_oszillator\", \"stoch_oszillator_signal\"]\n",
    "    }\n",
    "net_arch = {\"tiny\":[10,10], \"small\":[100,100], \"small+++\":[126,256], \"small++\":[256,126], \"small+\":[400,300], \"medium\":[300,300], \"large\":[500,500]}\n",
    "activation_fn = {\"tanh\": nn.Tanh, \"relu\": nn.ReLU, \"elu\": nn.ELU, \"leaky_relu\": nn.LeakyReLU}\n",
    "\n",
    "\n",
    "def backtest_ddpg(hyperparams, total_iterations, eval_freq, load_model=False, scenario=None):\n",
    "    timer = int(time.time())\n",
    "\n",
    "    pp = pprint.PrettyPrinter()\n",
    "    pp.pprint(hyperparams)\n",
    "\n",
    "    \"\"\" First iteration \"\"\"\n",
    "    agent = ddpg_agent(lookback_window_size=hyperparams[\"window_size\"],\n",
    "                      features=hyperparams[\"features\"])\n",
    "    agent.generate_portfolio(year=\"2018\", synthetic=False, split=\"train\")\n",
    "    agent.set_weight_storage(hyperparams[\"buffer_size\"])\n",
    "    agent.set_batch_size(hyperparams[\"batch_size\"])\n",
    "    \n",
    "    hyperparams.update({\"env\": agent})\n",
    "    hyperparams[\"policy_kwargs\"].update(\n",
    "        {\"features_extractor_class\": CustomCNN_DDPG,\n",
    "        \"features_extractor_kwargs\": dict(features_dim=13, agent_env=agent)}        \n",
    "    )\n",
    "    \n",
    "    if load_model is not None:\n",
    "        # Wrap the env\n",
    "        env = Monitor(agent)\n",
    "        env = DummyVecEnv([lambda: env])\n",
    "        _ = env.reset()\n",
    "\n",
    "        # Load the model and overwrite env\n",
    "        model = DDPG.load(f\"models/ddpg/{load_model}\")\n",
    "        model.env = env\n",
    "\n",
    "    else:\n",
    "        logdir = f\"logs/ddpg/{scenario}\"\n",
    "        model = DDPG(\n",
    "            policy = \"CnnPolicy\", \n",
    "            env = hyperparams[\"env\"],\n",
    "            learning_rate = hyperparams[\"learning_rate\"],\n",
    "            buffer_size = hyperparams[\"buffer_size\"],\n",
    "            batch_size = hyperparams[\"batch_size\"],\n",
    "            learning_starts = hyperparams[\"learning_starts\"],\n",
    "            gamma = hyperparams[\"gamma\"],\n",
    "            tau = hyperparams[\"tau\"],\n",
    "            gradient_steps = hyperparams[\"gradient_steps\"],\n",
    "            tensorboard_log = logdir,\n",
    "            policy_kwargs = hyperparams[\"policy_kwargs\"],\n",
    "            seed = hyperparams[\"seed\"],\n",
    "            verbose = 1,\n",
    "        )\n",
    "\n",
    "    total_timesteps = total_iterations * agent._crash_length\n",
    "    \n",
    "    # Creating the custom EvalCallback\n",
    "    #eval_callback = custom_eval_callback_scenario3(hyperparams, total_timesteps, \"DDPG\", eval_freq)\n",
    "    eval_callback = custom_eval_callback_testset(hyperparams, total_timesteps, \"DDPG\", eval_freq)\n",
    "    model.learn(total_timesteps, tb_log_name=f\"{timer}\", callback=eval_callback)\n",
    "    \n",
    "    \"\"\" Second iteration \"\"\"\n",
    "    agent = ddpg_agent(lookback_window_size=hyperparams[\"window_size\"],\n",
    "                      features=hyperparams[\"features\"])\n",
    "    agent.generate_portfolio(year=\"2018\", synthetic=True, split=\"train\")\n",
    "    agent.set_weight_storage(hyperparams[\"buffer_size\"])\n",
    "    agent.set_batch_size(hyperparams[\"batch_size\"])\n",
    "    \n",
    "    model.replay_buffer = model.replay_buffer_class(\n",
    "                model.buffer_size,\n",
    "                model.observation_space,\n",
    "                model.action_space,\n",
    "                device=model.device,\n",
    "                n_envs=model.n_envs,\n",
    "                optimize_memory_usage=model.optimize_memory_usage,\n",
    "                **model.replay_buffer_kwargs,\n",
    "    )\n",
    "    # Overwriting the model parameters with the new ones\n",
    "    model.env.envs[0].env = agent\n",
    "\n",
    "    total_timesteps = total_iterations * agent._crash_length\n",
    "    \n",
    "    # Creating the custom EvalCallback\n",
    "    #eval_callback = custom_eval_callback_scenario3(hyperparams, total_timesteps, \"DDPG\", eval_freq)\n",
    "    eval_callback = custom_eval_callback_testset(hyperparams, total_timesteps, \"DDPG\", eval_freq)\n",
    "    model.learn(total_timesteps, tb_log_name=f\"{timer}\", callback=eval_callback)\n",
    "\n",
    "    \"\"\" Third iteration \"\"\"\n",
    "    agent = ddpg_agent(lookback_window_size=hyperparams[\"window_size\"],\n",
    "                      features=hyperparams[\"features\"])\n",
    "    agent.generate_portfolio(year=\"2021\", synthetic=False, split=\"train\")\n",
    "    agent.set_weight_storage(hyperparams[\"buffer_size\"])\n",
    "    agent.set_batch_size(hyperparams[\"batch_size\"])\n",
    "    \n",
    "    model.replay_buffer = model.replay_buffer_class(\n",
    "                model.buffer_size,\n",
    "                model.observation_space,\n",
    "                model.action_space,\n",
    "                device=model.device,\n",
    "                n_envs=model.n_envs,\n",
    "                optimize_memory_usage=model.optimize_memory_usage,\n",
    "                **model.replay_buffer_kwargs,\n",
    "    )\n",
    "    # Overwriting the model parameters with the new ones\n",
    "    model.env.envs[0].env = agent\n",
    "\n",
    "    total_timesteps = total_iterations * agent._crash_length\n",
    "    \n",
    "    # Creating the custom EvalCallback\n",
    "    #eval_callback = custom_eval_callback_scenario3(hyperparams, total_timesteps, \"DDPG\", eval_freq)\n",
    "    eval_callback = custom_eval_callback_testset(hyperparams, total_timesteps, \"DDPG\", eval_freq)\n",
    "    model.learn(total_timesteps, tb_log_name=f\"{timer}\", callback=eval_callback)\n",
    "\n",
    "    \"\"\" Fourth iteration \"\"\"\n",
    "    agent = ddpg_agent(lookback_window_size=hyperparams[\"window_size\"],\n",
    "                      features=hyperparams[\"features\"])\n",
    "    agent.generate_portfolio(year=\"2021\", synthetic=True, split=\"train\")\n",
    "    agent.set_weight_storage(hyperparams[\"buffer_size\"])\n",
    "    agent.set_batch_size(hyperparams[\"batch_size\"])\n",
    "    \n",
    "    model.replay_buffer = model.replay_buffer_class(\n",
    "                model.buffer_size,\n",
    "                model.observation_space,\n",
    "                model.action_space,\n",
    "                device=model.device,\n",
    "                n_envs=model.n_envs,\n",
    "                optimize_memory_usage=model.optimize_memory_usage,\n",
    "                **model.replay_buffer_kwargs,\n",
    "    )\n",
    "    # Overwriting the model parameters with the new ones\n",
    "    model.env.envs[0].env = agent\n",
    "\n",
    "    total_timesteps = total_iterations * agent._crash_length\n",
    "    \n",
    "    # Creating the custom EvalCallback\n",
    "    #eval_callback = custom_eval_callback_scenario3(hyperparams, total_timesteps, \"DDPG\", eval_freq)\n",
    "    eval_callback = custom_eval_callback_testset(hyperparams, total_timesteps, \"DDPG\", eval_freq)\n",
    "    model.learn(total_timesteps, tb_log_name=f\"{timer}\", callback=eval_callback)\n",
    "\n",
    "    \"\"\" Backtest \"\"\"\n",
    "    test_agent = ddpg_agent(lookback_window_size=hyperparams[\"window_size\"],\n",
    "                      features=hyperparams[\"features\"])\n",
    "    test_agent.generate_portfolio(year=\"2022\", synthetic=False, split=\"whole\")\n",
    "\n",
    "    actions = [np.ones(13)/13]\n",
    "    done = False\n",
    "    obs = test_agent.reset()\n",
    "    while done is False:\n",
    "        w, _ = model.predict(torch.from_numpy(obs).float(), deterministic=True)\n",
    "        obs, reward, done, info = test_agent.step(w)\n",
    "        action = F.softmax(torch.from_numpy(w), dim=0).numpy()\n",
    "        actions.append(action)\n",
    "    test_agent._weight_storage = actions\n",
    "    test_agent.render()\n",
    "    \n",
    "    return test_agent, model\n",
    "\n",
    "hyperparams = {\n",
    "        \"batch_size\": 131,\n",
    "        \"buffer_size\": 15000,\n",
    "        \"features\" : [\"close\", \"low\", \"high\"] + features_list[\"RSI_SMA\"],\n",
    "        \"gamma\": 0.98,\n",
    "        \"gradient_steps\": 2500,\n",
    "        \"learning_rate\": 0.0741856008814114,\n",
    "        \"learning_starts\" : 110,\n",
    "        \"seed\" : 278,\n",
    "        \"tau\": 0.005,\n",
    "        \"window_size\" : 155,\n",
    "        \"policy_kwargs\": dict(\n",
    "            net_arch=net_arch[\"tiny\"],\n",
    "            activation_fn=activation_fn[\"elu\"],\n",
    "        )}\n",
    "\n",
    "#total_iterations=50; eval_freq=10\n",
    "total_iterations=1; eval_freq=1\n",
    "scenario = f\"scenario3_nosoft-cnn_it{total_iterations}\"; load_model = None\n",
    "test_agent, model = backtest_ddpg(hyperparams, total_iterations=total_iterations, eval_freq=eval_freq, scenario=scenario, load_model=load_model)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Saving the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cryptoportfolio.tools.performance_measures import max_drawdown, sharpe_ratio\n",
    "\n",
    "# Save the results\n",
    "data = open(\"logs/baselines_performance_metrics.csv\", \"a\")\n",
    "data.write(\"\\n\")\n",
    "data.write(test_agent._name + \"_\" + scenario); data.write(\",\")\n",
    "data.write(\"2022\"); data.write(\",\")\n",
    "data.write(str(test_agent._lookback_window_size)); data.write(\",\")\n",
    "data.write(str(max_drawdown(test_agent._portfolio_values))); data.write(\",\")\n",
    "data.write(str(test_agent._portfolio_values[-1])); data.write(\",\")\n",
    "data.write(str(sharpe_ratio(test_agent._rates_of_return))); data.write(\",\")\n",
    "data.write(str(test_agent._sum_of_transaction_costs))\n",
    "data.close()\n",
    "\n",
    "# Save the results\n",
    "data = open(\"logs/baselines_portfolio_values.csv\", \"a\")   \n",
    "data.write(\"\\n\")\n",
    "data.write(test_agent._name + \"_\" + scenario); data.write(\";\")\n",
    "data.write(\"2022\"); data.write(\";\")\n",
    "data.write(str(test_agent._lookback_window_size)); data.write(\";\")\n",
    "data.write(str(list(test_agent._portfolio_values)))\n",
    "data.close()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Saving Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(f\"models/ddpg/{scenario}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Loading model and continuing learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from stable_baselines3.common.vec_env import DummyVecEnv\n",
    "from stable_baselines3.common.monitor import Monitor\n",
    "\n",
    "scenario = \"_scenario2_nosoft-cnn_it50\"\n",
    "\n",
    "agent = ddpg_agent(lookback_window_size=hyperparams[\"window_size\"],\n",
    "                      features=hyperparams[\"features\"])\n",
    "agent.generate_portfolio(year=\"2021\", synthetic=False, split=\"train\")\n",
    "agent.set_weight_storage(hyperparams[\"buffer_size\"])\n",
    "agent.set_batch_size(hyperparams[\"batch_size\"])\n",
    "\n",
    "env = Monitor(agent)\n",
    "env = DummyVecEnv([lambda: env])\n",
    "obs = env.reset()\n",
    "\n",
    "new_model = DDPG.load(f\"models/ddpg/{scenario}\")\n",
    "new_model.env = env\n",
    "\n",
    "total_iterations = 50\n",
    "total_timesteps = agent._crash_length * total_iterations\n",
    "new_model.learn(total_timesteps, tb_log_name=\"scenario2_training\", reset_num_timesteps=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_model.tensorboard_log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from stable_baselines3.common.vec_env import DummyVecEnv\n",
    "from stable_baselines3.common.monitor import Monitor\n",
    "\n",
    "agent = ddpg_agent(lookback_window_size=hyperparams[\"window_size\"],\n",
    "                      features=hyperparams[\"features\"])\n",
    "agent.generate_portfolio(year=\"2021\", synthetic=False, split=\"train\")\n",
    "agent.set_weight_storage(hyperparams[\"buffer_size\"])\n",
    "agent.set_batch_size(hyperparams[\"batch_size\"])\n",
    "\n",
    "model = DDPG(\n",
    "        policy = \"CnnPolicy\", \n",
    "        env = agent,\n",
    "        learning_rate = hyperparams[\"learning_rate\"],\n",
    "        buffer_size = hyperparams[\"buffer_size\"],\n",
    "        batch_size = hyperparams[\"batch_size\"],\n",
    "        learning_starts = hyperparams[\"learning_starts\"],\n",
    "        gamma = hyperparams[\"gamma\"],\n",
    "        tau = hyperparams[\"tau\"],\n",
    "        gradient_steps = hyperparams[\"gradient_steps\"],\n",
    "        policy_kwargs = hyperparams[\"policy_kwargs\"],\n",
    "        seed = hyperparams[\"seed\"]\n",
    "    )\n",
    "model.learn(1)\n",
    "\n",
    "new_model = DDPG.load(f\"models/ddpg/{scenario}\")\n",
    "new_model.env = model.env\n",
    "new_model.learn(100, tb_log_name=\"scenario2_training\", reset_num_timesteps=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scenario = \"scenario4_nosoft-cnn-mlp_it24\"\n",
    "model = PPO.load(f\"models/ppo/{scenario}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime, time\n",
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3 import DDPG\n",
    "from cryptoportfolio.rlagent.rlagent_ddpg import RLAgent as ddpg_agent\n",
    "from cryptoportfolio.rlagent.network import CustomCNN_DDPG\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import numpy as np\n",
    "import torch.nn as nn\n",
    "import torch\n",
    "import pprint\n",
    "import torch.nn as nn\n",
    "from stable_baselines3.common.buffers import RolloutBuffer\n",
    "\n",
    "from stable_baselines3.common.vec_env import DummyVecEnv\n",
    "from stable_baselines3.common.monitor import Monitor\n",
    "\n",
    "features_list = {\"None\": [],\n",
    "        \"RSI\": [\"RSI\"],\n",
    "        \"mcd\": [\"mcd\", \"mcd_signal\"],\n",
    "        \"SMA\": [\"SMA_50\", \"SMA_200\"],\n",
    "        \"RSI_mcd\": [\"RSI\", \"mcd\", \"mcd_signal\"],\n",
    "        \"RSI_SMA\": [\"RSI\", \"SMA_50\", \"SMA_200\"],\n",
    "        \"OSZ\": [\"stoch_oszillator\", \"stoch_oszillator_signal\"],\n",
    "        \"RSI_OSZ\": [\"RSI\", \"stoch_oszillator\", \"stoch_oszillator_signal\"],\n",
    "        \"Bollinger\": [\"Bollinger_middle\" ,\"Bollinger_low\" ,\"Bollinger_high\"],\n",
    "        \"Bollinger_mcd\": [\"mcd\", \"mcd_signal\", \"Bollinger_middle\" ,\"Bollinger_low\" ,\"Bollinger_high\"],\n",
    "        \"RSI_SMA_Bollinger\": [\"RSI\", \"SMA_50\", \"SMA_200\", \"Bollinger_middle\" ,\"Bollinger_low\" ,\"Bollinger_high\"],\n",
    "        \"mcd_OSZ\": [\"mcd\", \"mcd_signal\", \"stoch_oszillator\", \"stoch_oszillator_signal\"],\n",
    "        \"Everything\": [\"RSI\", \"SMA_50\", \"SMA_200\", \"Bollinger_middle\" ,\"Bollinger_low\" ,\"Bollinger_high\", \"mcd\", \"mcd_signal\", \"stoch_oszillator\", \"stoch_oszillator_signal\"]\n",
    "    }\n",
    "net_arch = {\"default\":[400,300], \"tiny\":[10,10], \"tiny+\":[10,10,10], \"small\":[100,100], \"medium\":[300,300], \"large\":[500,500]}\n",
    "activation_fn = {\"tanh\": nn.Tanh, \"relu\": nn.ReLU, \"elu\": nn.ELU, \"leaky_relu\": nn.LeakyReLU}\n",
    "\n",
    "scenario1 = \"ppo/scenario1_nosoft-cnn-mlp_it50\"\n",
    "scenario2 = \"ppo/scenario2_nosoft-cnn-mlp_it50\"\n",
    "scenario3 = \"ppo/scenario3_nosoft-cnn-mlp_it30\"\n",
    "scenario4 = \"ppo/scenario4_nosoft-cnn-mlp_it50\"\n",
    "\n",
    "hyperparams = {\n",
    "        \"batch_size\": 131,\n",
    "        \"buffer_size\": 15000,\n",
    "        \"features\" : [\"close\", \"low\", \"high\"] + features_list[\"RSI_SMA\"],\n",
    "        \"gamma\": 0.98,\n",
    "        \"gradient_steps\": 2500,\n",
    "        \"learning_rate\": 0.0741856008814114,\n",
    "        \"learning_starts\" : 110,\n",
    "        \"seed\" : 278,\n",
    "        \"tau\": 0.005,\n",
    "        \"window_size\" : 155,\n",
    "        \"policy_kwargs\": dict(\n",
    "            net_arch=net_arch[\"tiny\"],\n",
    "            activation_fn=activation_fn[\"elu\"],\n",
    "        )}\n",
    "\n",
    "# Backtest\n",
    "test_agent = ddpg_agent(lookback_window_size=hyperparams[\"window_size\"],\n",
    "                    features=hyperparams[\"features\"])\n",
    "test_agent.generate_portfolio(year=\"2022\", synthetic=False, split=\"whole\")\n",
    "\n",
    "# Wrap the env\n",
    "env = Monitor(test_agent)\n",
    "env = DummyVecEnv([lambda: env])\n",
    "_ = env.reset()\n",
    "\n",
    "# Loading the model\n",
    "scenario = \"scenario3_nosoft-cnn_it50\"\n",
    "model = DDPG.load(f\"models/ddpg/{scenario}\")\n",
    "#new_model.env = env\n",
    "\n",
    "actions = [np.ones(13)/13]\n",
    "done = False\n",
    "obs = test_agent.reset()\n",
    "while done is False:\n",
    "    w, _ = model.predict(torch.from_numpy(obs).float(), deterministic=True)\n",
    "    obs, reward, done, info = test_agent.step(w)\n",
    "    action = F.softmax(torch.from_numpy(w), dim=0).numpy()\n",
    "    actions.append(action)\n",
    "test_agent._weight_storage = actions\n",
    "test_agent.render()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime, time\n",
    "from stable_baselines3 import PPO\n",
    "from cryptoportfolio.rlagent.rlagent_ppo import RLAgent as ppo_agent\n",
    "from cryptoportfolio.rlagent.network import CustomCNN_PPO\n",
    "import torch.nn.functional as F\n",
    "from cryptoportfolio.rlagent.network import CustomActorCriticPolicy \n",
    "\n",
    "import numpy as np\n",
    "import torch.nn as nn\n",
    "import torch\n",
    "import pprint\n",
    "import torch.nn as nn\n",
    "\n",
    "from stable_baselines3.common.vec_env import DummyVecEnv\n",
    "from stable_baselines3.common.monitor import Monitor\n",
    "\n",
    "features_list = {\"None\": [],\n",
    "        \"RSI\": [\"RSI\"],\n",
    "        \"mcd\": [\"mcd\", \"mcd_signal\"],\n",
    "        \"SMA\": [\"SMA_50\", \"SMA_200\"],\n",
    "        \"RSI_mcd\": [\"RSI\", \"mcd\", \"mcd_signal\"],\n",
    "        \"RSI_SMA\": [\"RSI\", \"SMA_50\", \"SMA_200\"],\n",
    "        \"OSZ\": [\"stoch_oszillator\", \"stoch_oszillator_signal\"],\n",
    "        \"RSI_OSZ\": [\"RSI\", \"stoch_oszillator\", \"stoch_oszillator_signal\"],\n",
    "        \"Bollinger\": [\"Bollinger_middle\" ,\"Bollinger_low\" ,\"Bollinger_high\"],\n",
    "        \"Bollinger_mcd\": [\"mcd\", \"mcd_signal\", \"Bollinger_middle\" ,\"Bollinger_low\" ,\"Bollinger_high\"],\n",
    "        \"RSI_SMA_Bollinger\": [\"RSI\", \"SMA_50\", \"SMA_200\", \"Bollinger_middle\" ,\"Bollinger_low\" ,\"Bollinger_high\"],\n",
    "        \"mcd_OSZ\": [\"mcd\", \"mcd_signal\", \"stoch_oszillator\", \"stoch_oszillator_signal\"],\n",
    "        \"Everything\": [\"RSI\", \"SMA_50\", \"SMA_200\", \"Bollinger_middle\" ,\"Bollinger_low\" ,\"Bollinger_high\", \"mcd\", \"mcd_signal\", \"stoch_oszillator\", \"stoch_oszillator_signal\"]\n",
    "    }\n",
    "activation_fn = {\"tanh\": nn.Tanh, \"relu\": nn.ReLU, \"elu\": nn.ELU, \"leaky_relu\": nn.LeakyReLU}\n",
    "\n",
    "scenario1 = \"ppo/scenario1_nosoft-cnn-mlp_it50\"\n",
    "scenario2 = \"ppo/scenario2_nosoft-cnn-mlp_it50\"\n",
    "scenario3 = \"ppo/scenario3_nosoft-cnn-mlp_it30\"\n",
    "scenario4 = \"ppo/scenario4_nosoft-cnn-mlp_it50\"\n",
    "\n",
    "hyperparams = {\n",
    "        \"policy\": CustomActorCriticPolicy, \n",
    "        \"clip_range\": 0.1,\n",
    "        \"ent_coef\" : 0.0028646621529157634,\n",
    "        \"features\" : [\"close\", \"low\", \"high\"] + features_list[\"RSI_SMA_Bollinger\"],\n",
    "        \"gae_lambda\": 0.99,\n",
    "        \"gamma\": 0.95,\n",
    "        \"learning_rate\" : 0.0001735302876108352,\n",
    "        \"max_grad_norm\" : 5.0,\n",
    "        \"n_epochs\" : 25,\n",
    "        \"seed\" : 1028,\n",
    "        \"vf_coef\" : 0.004473747041794429,\n",
    "        \"window_size\" : 150,\n",
    "        \"batch_size\" : 15,\n",
    "        \"policy_kwargs\": dict(\n",
    "            layer_size=256,\n",
    "            n_layers=3,\n",
    "            activation_fn=activation_fn[\"tanh\"],\n",
    "            ortho_init=False,\n",
    "        )}\n",
    "\n",
    "# Backtest\n",
    "test_agent = ppo_agent(lookback_window_size=hyperparams[\"window_size\"],\n",
    "                    features=hyperparams[\"features\"],\n",
    "                    batch_size=hyperparams[\"batch_size\"])\n",
    "test_agent.generate_portfolio(year=\"2022\", synthetic=False, split=\"whole\")\n",
    "\n",
    "# Loading the model\n",
    "scenario = \"scenario3_nosoft-cnn-mlp_it30\"\n",
    "model = PPO.load(f\"models/ppo/{scenario}\")\n",
    "#new_model.env = env\n",
    "\n",
    "done = False\n",
    "obs = test_agent.reset()\n",
    "while done is False:\n",
    "    w, _ = model.predict(torch.from_numpy(obs).float(), deterministic=True)\n",
    "    obs, reward, done, info = test_agent.step(w)\n",
    "test_agent.render()"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "fb833273add3e7c60eb33c0608260b79a61e072ade6f02cc8d07b0a26eef8ab8"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}