9.2 - Setup Monitors DeepAR.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part IX.2 - Configure Model Quality Monitor for Deep AR\n",
    "\n",
    "University of San Diego - MS Applied AI\n",
    "\n",
    "AAI-540 Team 5\n",
    "\n",
    "October 21, 2024"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /home/sagemaker-user/.config/sagemaker/config.yaml\n",
      "Stored 's3_datalake_path_csv' (str)\n",
      "Stored 'local_data_path_csv' (str)\n",
      "Stored 's3_datalake_path_parquet' (str)\n"
     ]
    }
   ],
   "source": [
    "# setup environment\n",
    "%run 0-Environment_Setup.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import necessary libraries\n",
    "from sagemaker import image_uris\n",
    "from sagemaker.model import Model\n",
    "from sagemaker.model_monitor import DataCaptureConfig\n",
    "from sagemaker.model_monitor import DefaultModelMonitor\n",
    "from sagemaker.model_monitor import ModelQualityMonitor\n",
    "from sagemaker.model_monitor import EndpointInput\n",
    "from sagemaker.model_monitor.dataset_format import DatasetFormat\n",
    "from datetime import datetime, timedelta, timezone\n",
    "from sagemaker.base_serializers import JSONSerializer, IdentitySerializer\n",
    "from sagemaker.base_deserializers import JSONDeserializer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Demo Bucket: sagemaker-us-east-1-343218227212\n",
      "Image URI: 156813124566.dkr.ecr.us-east-1.amazonaws.com/sagemaker-model-monitor-analyzer\n",
      "Capture path: s3://sagemaker-us-east-1-343218227212/sagemaker/deepAR-sales-forecasting-ModelQualityMonitor/datacapture\n",
      "Ground truth path: s3://sagemaker-us-east-1-343218227212/sagemaker/deepAR-sales-forecasting-ModelQualityMonitor/ground_truth_data/2024-10-13-20-30-01\n",
      "Report path: s3://sagemaker-us-east-1-343218227212/sagemaker/deepAR-sales-forecasting-ModelQualityMonitor/reports\n"
     ]
    }
   ],
   "source": [
    "# Setup S3 bucket\n",
    "bucket = sess.default_bucket()\n",
    "print(\"Demo Bucket:\", bucket)\n",
    "prefix = \"sagemaker/deepAR-sales-forecasting-ModelQualityMonitor\"\n",
    "\n",
    "##S3 prefixes\n",
    "data_capture_prefix = f\"{prefix}/datacapture\"\n",
    "s3_capture_upload_path = f\"s3://{bucket}/{data_capture_prefix}\"\n",
    "\n",
    "ground_truth_upload_path = (\n",
    "    f\"s3://{bucket}/{prefix}/ground_truth_data/{datetime.now():%Y-%m-%d-%H-%M-%S}\"\n",
    ")\n",
    "\n",
    "reports_prefix = f\"{prefix}/reports\"\n",
    "s3_report_path = f\"s3://{bucket}/{reports_prefix}\"\n",
    "\n",
    "##Get the model monitor image\n",
    "monitor_image_uri = image_uris.retrieve(framework=\"model-monitor\", region=region)\n",
    "\n",
    "print(\"Image URI:\", monitor_image_uri)\n",
    "print(f\"Capture path: {s3_capture_upload_path}\")\n",
    "print(f\"Ground truth path: {ground_truth_upload_path}\")\n",
    "print(f\"Report path: {s3_report_path}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setup dedicated endpoint for our monitoring using our best model\n",
    "\n",
    "Here, we will deploy our best deepAR model found during our hyperparameter tuning process to a real time endpoint.  A model quality monitor will be attached to this endpoint.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"deepar-hyperparamete-241007-2220-007-best\"\n",
    "image_uri = sagemaker.image_uris.retrieve(\"forecasting-deepar\", region)\n",
    "model_url = \"s3://sagemaker-us-east-1-343218227212/store-sales-forecasting/deepar/gold-dataset/output/deepar-hyperparamete-241007-2220-007-b18b6b1e/output/model.tar.gz\"\n",
    "endpoint_name = 'deepar-store-sales-forecasting-model-quality-monitor'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Endpoint Configuration\n",
    "model = Model(image_uri=image_uri, model_data=model_url, role=role, sagemaker_session=sess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Deploy our best model to a monitoring endpoint\n",
    "print(\"EndpointName =\", endpoint_name)\n",
    "\n",
    "data_capture_config = DataCaptureConfig(\n",
    "    enable_capture=True, sampling_percentage=100, destination_s3_uri=s3_capture_upload_path\n",
    ")\n",
    "\n",
    "model.deploy(\n",
    "    initial_instance_count=1,\n",
    "    instance_type=\"ml.m4.xlarge\",\n",
    "    endpoint_name=endpoint_name,\n",
    "    data_capture_config=data_capture_config,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The class RealTimePredictor has been renamed in sagemaker>=2.\n",
      "See: https://sagemaker.readthedocs.io/en/stable/v2.html for details.\n",
      "content_type is a no-op in sagemaker>=2.\n",
      "See: https://sagemaker.readthedocs.io/en/stable/v2.html for details.\n"
     ]
    }
   ],
   "source": [
    "# create predictor for running predictions through endpoint\n",
    "json_serializer = JSONSerializer()\n",
    "predictor = sagemaker.predictor.RealTimePredictor(\n",
    "    endpoint_name, \n",
    "    sagemaker_session=sess, \n",
    "    serializer=IdentitySerializer(content_type=\"application/json\"),\n",
    "    deserializer=JSONDeserializer(),\n",
    "    content_type=\"application/json\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Helper Functions for Basline Jobs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# helper function to transform test dataset to ground truth data frame\n",
    "def create_ground_truths_from_test(test_data):\n",
    "    ground_truths = []\n",
    "    start_date = pd.to_datetime(test_data[0]['start'])\n",
    "    for sample in test_data:\n",
    "        if(pd.to_datetime(sample['start']) > start_date):\n",
    "            for idx, daily_ground_truth in enumerate(sample['target']):\n",
    "                current_date = pd.to_datetime(sample['start']) + timedelta(days=idx)\n",
    "                ground_truths.append([current_date, sample['cat'][0] + 1, daily_ground_truth, 0])\n",
    "    \n",
    "    ground_truth_df = pd.DataFrame(columns=['date', 'store_nbr', 'label', 'prediction'], data=ground_truths)\n",
    "    ground_truth_df['date'] = pd.to_datetime(ground_truth_df['date'])\n",
    "    return ground_truth_df\n",
    "\n",
    "# run predictions through the end point\n",
    "def run_predictions_on_dataset(test_data_input, test_ground_truth_df):\n",
    "    for idx, model_input in enumerate(test_data_input):\n",
    "        start_date = model_input['start']\n",
    "        result = predictor.predict(create_deepar_input_from_test(model_input))\n",
    "        model_predictions = result['predictions'][0]['mean']\n",
    "        for day_idx, prediction in enumerate(model_predictions):\n",
    "            test_ground_truth_df.loc[((idx * 7) + day_idx), 'prediction'] = round(prediction, 2)\n",
    "    return test_ground_truth_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run job to generate baseline for model quality\n",
    "\n",
    "The baseline job will run traffic through our endpoint and compare with the expected (ground truth) data in order to establish a baseline of metrics for our regression task.  In order to faciliate this, some additional steps are required:\n",
    "\n",
    "1.  Generate a ground truth from our test dataset - this entails pulling down our test dataset, and \"flattening\" the weekly sequences of per day sales into individual rows by day.\n",
    "2.  Second, the test dataset will be fed through the model\n",
    "3.  The predictions from the model are post-processed and aligned to the appropriate row (keyed by date) alongside the appropriate ground truth\n",
    "4.  This consolidated ground truth+predictions are used in the baseline job to calculate baseline metrics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "# helper function to transform test dataset to ground truth data frame\n",
    "def create_ground_truths_from_test(test_data):\n",
    "    ground_truths = []\n",
    "    start_date = pd.to_datetime(test_data[0]['start'])\n",
    "    for sample in test_data:\n",
    "        if(pd.to_datetime(sample['start']) > start_date):\n",
    "            for idx, daily_ground_truth in enumerate(sample['target']):\n",
    "                current_date = pd.to_datetime(sample['start']) + timedelta(days=idx)\n",
    "                ground_truths.append([current_date, sample['cat'][0] + 1, daily_ground_truth, 0])\n",
    "    \n",
    "    ground_truth_df = pd.DataFrame(columns=['date', 'store_nbr', 'label', 'prediction'], data=ground_truths)\n",
    "    ground_truth_df['date'] = pd.to_datetime(ground_truth_df['date'])\n",
    "    return ground_truth_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline data uri: s3://sagemaker-us-east-1-343218227212/sagemaker/deepAR-sales-forecasting-ModelQualityMonitor/baselining/data\n",
      "Baseline results uri: s3://sagemaker-us-east-1-343218227212/sagemaker/deepAR-sales-forecasting-ModelQualityMonitor/baselining/results\n"
     ]
    }
   ],
   "source": [
    "# setup variables\n",
    "input_data_path = \"{}/test/test.json\".format(s3_deepar_gold_dataset_path)\n",
    "inference_filename = \"test.json\"\n",
    "input_local_filename = f\"test-data/{inference_filename}\"\n",
    "\n",
    "baseline_prefix = prefix + \"/baselining\"\n",
    "baseline_data_prefix = baseline_prefix + \"/data\"\n",
    "baseline_results_prefix = baseline_prefix + \"/results\"\n",
    "\n",
    "baseline_data_uri = f\"s3://{bucket}/{baseline_data_prefix}\"\n",
    "baseline_results_uri = f\"s3://{bucket}/{baseline_results_prefix}\"\n",
    "print(f\"Baseline data uri: {baseline_data_uri}\")\n",
    "print(f\"Baseline results uri: {baseline_results_uri}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "download: s3://sagemaker-us-east-1-343218227212/store-sales-forecasting/deepar/gold-dataset/test/test.json to test-data/test.json\n"
     ]
    }
   ],
   "source": [
    "# copy validation dataset local\n",
    "!aws s3 cp $input_data_path $input_local_filename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load test data\n",
    "test_input = load_json_by_line(input_local_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>store_nbr</th>\n",
       "      <th>label</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2017-03-06</td>\n",
       "      <td>1</td>\n",
       "      <td>10819.11</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2017-03-07</td>\n",
       "      <td>1</td>\n",
       "      <td>14300.59</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2017-03-08</td>\n",
       "      <td>1</td>\n",
       "      <td>13247.73</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2017-03-09</td>\n",
       "      <td>1</td>\n",
       "      <td>10605.34</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2017-03-10</td>\n",
       "      <td>1</td>\n",
       "      <td>12329.87</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7933</th>\n",
       "      <td>2017-07-26</td>\n",
       "      <td>54</td>\n",
       "      <td>6572.49</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7934</th>\n",
       "      <td>2017-07-27</td>\n",
       "      <td>54</td>\n",
       "      <td>6911.35</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7935</th>\n",
       "      <td>2017-07-28</td>\n",
       "      <td>54</td>\n",
       "      <td>9531.09</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7936</th>\n",
       "      <td>2017-07-29</td>\n",
       "      <td>54</td>\n",
       "      <td>10878.75</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7937</th>\n",
       "      <td>2017-07-30</td>\n",
       "      <td>54</td>\n",
       "      <td>15815.71</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7938 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           date  store_nbr     label  prediction\n",
       "0    2017-03-06          1  10819.11           0\n",
       "1    2017-03-07          1  14300.59           0\n",
       "2    2017-03-08          1  13247.73           0\n",
       "3    2017-03-09          1  10605.34           0\n",
       "4    2017-03-10          1  12329.87           0\n",
       "...         ...        ...       ...         ...\n",
       "7933 2017-07-26         54   6572.49           0\n",
       "7934 2017-07-27         54   6911.35           0\n",
       "7935 2017-07-28         54   9531.09           0\n",
       "7936 2017-07-29         54  10878.75           0\n",
       "7937 2017-07-30         54  15815.71           0\n",
       "\n",
       "[7938 rows x 4 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create the ground truth dataframe\n",
    "test_ground_truth_df = create_ground_truths_from_test(test_input)\n",
    "test_ground_truth_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_26464/940235236.py:22: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise an error in a future version of pandas. Value '8658.6' has dtype incompatible with int64, please explicitly cast to a compatible dtype first.\n",
      "  test_ground_truth_df.loc[((idx * 7) + day_idx), 'prediction'] = round(prediction, 2)\n"
     ]
    }
   ],
   "source": [
    "# run predictions to generate ground truth\n",
    "ground_truth_df = run_predictions_on_dataset(test_input, test_ground_truth_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# store the copy to CSV\n",
    "test_ground_truth_df_sub = ground_truth_df[['prediction', 'label']]\n",
    "test_ground_truth_df_sub = test_ground_truth_df_sub[:200]\n",
    "test_ground_truth_df_sub.to_csv(\"test-data/test_with_predictions.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prediction,label\n",
      "8658.6,10819.11\n",
      "8072.99,14300.59\n",
      "11284.56,13247.73\n",
      "7785.69,10605.34\n",
      "8636.93,12329.87\n",
      "7246.98,11997.51\n",
      "6906.86,5897.18\n",
      "16384.1,13852.91\n",
      "16360.5,13113.42\n"
     ]
    }
   ],
   "source": [
    "!head test-data/test_with_predictions.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting existing file\n",
      "Uploading file to s3://sagemaker-us-east-1-343218227212/sagemaker/deepAR-sales-forecasting-ModelQualityMonitor/baselining/data/test_with_predictions.csv\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'s3://sagemaker-us-east-1-343218227212/sagemaker/deepAR-sales-forecasting-ModelQualityMonitor/baselining/data/test_with_predictions.csv'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# upload to S3\n",
    "baseline_dataset_uri = \"{}/test_with_predictions.csv\".format(baseline_data_uri)\n",
    "copy_to_s3(f\"test-data/test_with_predictions.csv\", baseline_dataset_uri, True)\n",
    "baseline_dataset_uri"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Baselining Job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:sagemaker.image_uris:Defaulting to the only supported framework/algorithm version: .\n",
      "INFO:sagemaker.image_uris:Ignoring unnecessary instance type: None.\n"
     ]
    }
   ],
   "source": [
    "# Create the model quality monitoring object\n",
    "deepar_model_quality_monitor = ModelQualityMonitor(\n",
    "    role=role,\n",
    "    instance_count=1,\n",
    "    instance_type=\"ml.m5.2xlarge\",\n",
    "    volume_size_in_gb=20,\n",
    "    max_runtime_in_seconds=1800,\n",
    "    sagemaker_session=sess,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Name of the model quality baseline job\n",
    "baseline_job_name = f\"deepar-store-sales-predictions-baseline-job-{datetime.utcnow():%Y-%m-%d-%H%M}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:sagemaker:Creating processing-job with name deepar-store-sales-predictions-baseline-job-2024-10-13-2041\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "............................................................!"
     ]
    }
   ],
   "source": [
    "# Execute the baseline suggestion job.\n",
    "job = deepar_model_quality_monitor.suggest_baseline(\n",
    "    job_name=baseline_job_name,\n",
    "    baseline_dataset=baseline_dataset_uri,\n",
    "    dataset_format=DatasetFormat.csv(header=True),\n",
    "    output_s3_uri=baseline_results_uri,\n",
    "    problem_type=\"Regression\",\n",
    "    inference_attribute=\"prediction\",\n",
    "    ground_truth_attribute=\"label\",\n",
    ")\n",
    "job.wait(logs=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inspect results and thresholds from baseline job\n",
    "\n",
    "Visualize and inspect the results of our baseline job "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the latest baseline job\n",
    "baseline_job = deepar_model_quality_monitor.latest_baselining_job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mae.value</th>\n",
       "      <td>2.620179e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mae.standard_deviation</th>\n",
       "      <td>5.521575e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mse.value</th>\n",
       "      <td>1.556714e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mse.standard_deviation</th>\n",
       "      <td>8.732826e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rmse.value</th>\n",
       "      <td>3.945522e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rmse.standard_deviation</th>\n",
       "      <td>1.119801e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>r2.value</th>\n",
       "      <td>7.555614e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>r2.standard_deviation</th>\n",
       "      <td>1.284822e-02</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    0\n",
       "mae.value                2.620179e+03\n",
       "mae.standard_deviation   5.521575e+01\n",
       "mse.value                1.556714e+07\n",
       "mse.standard_deviation   8.732826e+05\n",
       "rmse.value               3.945522e+03\n",
       "rmse.standard_deviation  1.119801e+02\n",
       "r2.value                 7.555614e-01\n",
       "r2.standard_deviation    1.284822e-02"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# inspect the baseline metrics\n",
    "regression_metrics = baseline_job.baseline_statistics().body_dict[\"regression_metrics\"]\n",
    "pd.json_normalize(regression_metrics).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>threshold</th>\n",
       "      <th>comparison_operator</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mae</th>\n",
       "      <td>2620.1789</td>\n",
       "      <td>GreaterThanThreshold</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mse</th>\n",
       "      <td>15567140.995764</td>\n",
       "      <td>GreaterThanThreshold</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rmse</th>\n",
       "      <td>3945.521638</td>\n",
       "      <td>GreaterThanThreshold</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>r2</th>\n",
       "      <td>0.755561</td>\n",
       "      <td>LessThanThreshold</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            threshold   comparison_operator\n",
       "mae         2620.1789  GreaterThanThreshold\n",
       "mse   15567140.995764  GreaterThanThreshold\n",
       "rmse      3945.521638  GreaterThanThreshold\n",
       "r2           0.755561     LessThanThreshold"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(baseline_job.suggested_constraints().body_dict[\"regression_constraints\"]).T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate some traffic and observe\n",
    "\n",
    "In this section, we will generate traffic to our endpoint using our test dataset.  This will result in capture files being generated to be used withour ModelMonitor."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker.s3 import S3Downloader, S3Uploader\n",
    "from time import sleep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run predictions on all test samples\n",
    "ground_truth_df = run_predictions_on_dataset(test_input, test_ground_truth_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for captures to show up........................................................................................................................\n",
      "Found Capture Files:\n",
      "s3://sagemaker-us-east-1-343218227212/sagemaker/deepAR-sales-forecasting-ModelQualityMonitor/datacapture/deepar-store-sales-forecasting-model-quality-monitor/AllTraffic/2024/10/14/00/50-51-369-a6156a76-8a40-4560-bdca-2d5e7f3066a3.jsonl\n",
      " s3://sagemaker-us-east-1-343218227212/sagemaker/deepAR-sales-forecasting-ModelQualityMonitor/datacapture/deepar-store-sales-forecasting-model-quality-monitor/AllTraffic/2024/10/14/01/21-04-526-4a6f5d4e-3efc-44cb-8df7-878eac82f557.jsonl\n",
      " s3://sagemaker-us-east-1-343218227212/sagemaker/deepAR-sales-forecasting-ModelQualityMonitor/datacapture/deepar-store-sales-forecasting-model-quality-monitor/AllTraffic/2024/10/14/02/24-28-897-ae055824-ce1c-4777-ba48-40263696ab62.jsonl\n"
     ]
    }
   ],
   "source": [
    "# wait for capture files\n",
    "print(\"Waiting for captures to show up\", end=\"\")\n",
    "for _ in range(120):\n",
    "    capture_files = sorted(S3Downloader.list(f\"{s3_capture_upload_path}/{endpoint_name}\"))\n",
    "    if capture_files:\n",
    "        capture_file = S3Downloader.read_file(capture_files[-1]).split(\"\\n\")\n",
    "        capture_record = json.loads(capture_file[0])\n",
    "        if \"inferenceId\" in capture_record[\"eventMetadata\"]:\n",
    "            break\n",
    "    print(\".\", end=\"\", flush=True)\n",
    "    sleep(1)\n",
    "print()\n",
    "print(\"Found Capture Files:\")\n",
    "print(\"\\n \".join(capture_files[-3:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"captureData\":{\"endpointInput\":{\"observedContentType\":\"application/json\",\"mode\":\"INPUT\",\"data\":\"{\\\"instances\\\": [{\\\"start\\\": \\\"2017-07-24 00:00:00\\\", \\\"target\\\": [11461.16, 8396.32, 12397.49, 7726.76, 11183.51, 14961.54, 17141.29], \\\"cat\\\": [52], \\\"dynamic_feat\\\": [[46.21, 47.77, 48.58, 49.05, 49.72, 49.72, 49.72, 50.21, 49.19, 49.6, 49.03, 49.57, 49.57, 49.57], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [72, 65, 171, 48, 0, 57, 55, 58, 46, 181, 31, 50, 37, 36]]}], \\\"configuration\\\": {\\\"output_types\\\": [\\\"mean\\\"]}}\",\"encoding\":\"JSON\"},\"endpointOutput\":{\"observedContentType\":\"application/json\",\"mode\":\"OUTPUT\",\"data\":\"{\\\"predictions\\\":[{\\\"mean\\\":[11406.5859375,12286.6484375,13242.875,9363.9501953125,10761.6123046875,12092.767578125,11646.5283203125]}]}\",\"encoding\":\"JSON\"}},\"eventMetadata\":{\"eventId\":\"fc4f79e0-67c3-4848-a796-1c586fb215cc\",\"inferenceTime\":\"2024-10-14T02:25:02Z\"},\"eventVersion\":\"0\"}\n",
      "{\"captureData\":{\"endpointInput\":{\"observedContentType\":\"application/json\",\"mode\":\"INPUT\",\"data\":\"{\\\"instances\\\": [{\\\"start\\\": \\\"2017-07-24 00:00:00\\\", \\\"target\\\": [10245.25, 8241.91, 6572.49, 6911.35, 9531.09, 10878.75, 15815.71], \\\"cat\\\": [53], \\\"dynamic_feat\\\": [[46.21, 47.77, 48.58, 49.05, 49.72, 49.72, 49.72, 50.21, 49.19, 49.6, 49.03, 49.57, 49.57, 49.57], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [50, 75, 65, 53, 52, 51, 59, 49, 81, 64, 50, 53, 58, 59]]}], \\\"configuration\\\": {\\\"output_types\\\": [\\\"mean\\\"]}}\",\"encoding\":\"JSON\"},\"endpointOutput\":{\"observedContentType\":\"application/json\",\"mode\":\"OUTPUT\",\"data\":\"{\\\"predictions\\\":[{\\\"mean\\\":[9087.20703125,9731.0146484375,7777.177734375,8009.11328125,8057.1875,9041.5791015625,9013.998046875]}]}\",\"encoding\":\"JSON\"}},\"eventMetadata\":{\"eventId\":\"4372f017-453b-4da7-87b6-0f28765ee87e\",\"inferenceTime\":\"2024-10-14T02:25:02Z\"},\"eventVersion\":\"0\"}\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n\".join(capture_file[-3:-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"captureData\": {\n",
      "    \"endpointInput\": {\n",
      "      \"observedContentType\": \"application/json\",\n",
      "      \"mode\": \"INPUT\",\n",
      "      \"data\": \"{\\\"instances\\\": [{\\\"start\\\": \\\"2017-02-27 00:00:00\\\", \\\"target\\\": [2951.04, 5553.76, 16819.46, 12781.88, 13219.13, 12086.53, 5156.63], \\\"cat\\\": [0], \\\"dynamic_feat\\\": [[54.04, 54.0, 53.82, 52.63, 53.33, 53.33, 53.33, 53.19, 52.68, 49.83, 48.75, 48.05, 48.05, 48.05], [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [52, 53, 216, 52, 56, 55, 41, 39, 41, 223, 28, 53, 31, 23]]}], \\\"configuration\\\": {\\\"output_types\\\": [\\\"mean\\\"]}}\",\n",
      "      \"encoding\": \"JSON\"\n",
      "    },\n",
      "    \"endpointOutput\": {\n",
      "      \"observedContentType\": \"application/json\",\n",
      "      \"mode\": \"OUTPUT\",\n",
      "      \"data\": \"{\\\"predictions\\\":[{\\\"mean\\\":[8505.0751953125,8233.1552734375,11329.0625,7039.3139648438,9364.3828125,7480.6567382812,5257.5043945312]}]}\",\n",
      "      \"encoding\": \"JSON\"\n",
      "    }\n",
      "  },\n",
      "  \"eventMetadata\": {\n",
      "    \"eventId\": \"bab21866-2526-42a0-9579-c87a1313c940\",\n",
      "    \"inferenceTime\": \"2024-10-14T02:24:28Z\"\n",
      "  },\n",
      "  \"eventVersion\": \"0\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "print(json.dumps(capture_record, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate Ground Truth\n",
    "\n",
    "This section will generate ground truth dataset from our latest capture and upload to S3 in preparation for next ModelMonitor run.  In addition, the capture file will be tranformed to match the ground truth format.  This is largely due to challenges getting the monitor merge job to work with the JSON output from our model.  To work around this, the ground truth generation and capture output will do the following:\n",
    "\n",
    "1.  Pull down latest capture file\n",
    "2.  Generate a ground truth file for the first 201 capture records - using 1 day of the prediction horizon - and setting format to CSV (instead of JSON)\n",
    "3.  Transform the capture file to also format to CSV and include only 1 day of the prediction horizon to align with ground truth.  Only include the first 201 records.\n",
    "4.  Push capture file back to s3, overwriting original capture file\n",
    "5.  Push ground truth data to s3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!mkdir code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%writefile code/preprocess.py\n",
    "# import json\n",
    "# def preprocess_handler(inference_record, logger):\n",
    "#     logger.info(\"[Preprocessor Script Starting]\")\n",
    "#     logger.info(f\"[Preprocessor Record]: {inference_record}\")\n",
    "#     output_data = json.loads(inference_record.endpoint_output.data)\n",
    "#     logger.info(f\"[Preprocessor Output Chunk 1]: {output_data}\")\n",
    "#     output_data = output_data['predictions'][0]['mean']\n",
    "#     logger.info(f\"[Preprocessor Output Chunk 2]: {output_data}\")\n",
    "#     logger.info(f\"[Preprocessor Output Chunk 2]: {str(output_data[0])}\")\n",
    "#     return json.dumps({ \"0\" : str(output_data[0]) })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [],
   "source": [
    "# helper function to generate ground truth data from our model capture\n",
    "def generate_ground_truth_from_capture(model_inputs, capture_outputs):\n",
    "    capture_idx = 0\n",
    "    ground_truths = []\n",
    "    start_date = pd.to_datetime(model_inputs[0]['start'])\n",
    "    for model_input in model_inputs:\n",
    "        #print(str(model_input['target']))\n",
    "        if(pd.to_datetime(model_input['start']) > start_date):\n",
    "            capture = capture_outputs[capture_idx]\n",
    "            event_id = capture[\"eventMetadata\"][\"eventId\"]\n",
    "            ground_truth = {\n",
    "                \"groundTruthData\": {\n",
    "                    \"data\": str(model_input['target'][0]),\n",
    "                    \"encoding\": \"CSV\",\n",
    "                },\n",
    "                \"eventMetadata\": {\n",
    "                    \"eventId\": str(event_id),\n",
    "                },\n",
    "                \"eventVersion\": \"0\",\n",
    "            }\n",
    "            ground_truths.append(ground_truth)\n",
    "            capture_idx += 1\n",
    "            if(capture_idx > 200):\n",
    "                break\n",
    "\n",
    "    return ground_truths\n",
    "\n",
    "# helper function to simplify capture data to a single prediction\n",
    "def transofrm_capture_data(capture_data, multiplier):\n",
    "    capture_idx = 0\n",
    "    transformed_captures = []\n",
    "    for idx, capture in enumerate(capture_data):\n",
    "        capture_data = json.loads(capture['captureData']['endpointOutput']['data'])\n",
    "        capture['captureData']['endpointOutput']['data'] = str(float(capture_data['predictions'][0]['mean'][0]) * multiplier)\n",
    "        capture['captureData']['endpointOutput']['encoding'] = \"CSV\"\n",
    "        capture['captureData']['endpointOutput']['observedContentType'] = \"text/csv; charset-utf-8\"        \n",
    "        transformed_captures.append(capture)\n",
    "        capture_idx += 1\n",
    "        if(capture_idx > 200):\n",
    "            break\n",
    "    return transformed_captures\n",
    "\n",
    "# helper function to upload ground truth data\n",
    "def upload_ground_truth(records, upload_time):\n",
    "    fake_records = [json.dumps(r) for r in records]\n",
    "    data_to_upload = \"\\n\".join(fake_records)\n",
    "    target_s3_uri = f\"{ground_truth_upload_path}/{upload_time:%Y/%m/%d/%H/%M%S}.jsonl\"\n",
    "    print(f\"Uploading {len(fake_records)} records to\", target_s3_uri)\n",
    "    S3Uploader.upload_string_as_file_body(data_to_upload, target_s3_uri)\n",
    "\n",
    "# helper function to upload ground truth data\n",
    "def upload_transformed_capture(records, capture_file_path):\n",
    "    fake_records = [json.dumps(r) for r in records]\n",
    "    data_to_upload = \"\\n\".join(fake_records)\n",
    "    target_s3_uri = capture_file_path\n",
    "    print(f\"Uploading {len(fake_records)} records to\", target_s3_uri)\n",
    "    S3Uploader.upload_string_as_file_body(data_to_upload, target_s3_uri)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get latest capture file\n",
    "latest_capture_file = capture_files[-1]\n",
    "capture_local_filename = f\"test-data/latest-capture.jsonl\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "download: s3://sagemaker-us-east-1-343218227212/sagemaker/deepAR-sales-forecasting-ModelQualityMonitor/datacapture/deepar-store-sales-forecasting-model-quality-monitor/AllTraffic/2024/10/14/02/24-28-897-ae055824-ce1c-4777-ba48-40263696ab62.jsonl to test-data/latest-capture.jsonl\n"
     ]
    }
   ],
   "source": [
    "# copy capture file to local storage for processing\n",
    "!aws s3 cp $latest_capture_file $capture_local_filename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load capture file into JSON\n",
    "capture_data = load_json_by_line(capture_local_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate ground truth data\n",
    "capture_ground_truth = generate_ground_truth_from_capture(test_input, capture_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert capture file to CSV with only one prediction\n",
    "capture_data_transformed = transofrm_capture_data(capture_data, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Uploading 201 records to s3://sagemaker-us-east-1-343218227212/sagemaker/deepAR-sales-forecasting-ModelQualityMonitor/ground_truth_data/2024-10-13-20-30-01/2024/10/14/02/2828.jsonl\n"
     ]
    }
   ],
   "source": [
    "# upload ground truth\n",
    "upload_ground_truth(capture_ground_truth, datetime.utcnow())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Uploading 201 records to s3://sagemaker-us-east-1-343218227212/sagemaker/deepAR-sales-forecasting-ModelQualityMonitor/datacapture/deepar-store-sales-forecasting-model-quality-monitor/AllTraffic/2024/10/14/02/24-28-897-ae055824-ce1c-4777-ba48-40263696ab62.jsonl\n"
     ]
    }
   ],
   "source": [
    "# upload transformed capture\n",
    "upload_transformed_capture(capture_data_transformed, latest_capture_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Monitoring Schedule and Inspect Execution Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "# monitoring schedule name\n",
    "deepar_monitor_schedule_name = (\n",
    "    f\"deepar-sales-forecasting-model-monitoring-schedule-12\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create an enpointInput\n",
    "endpoint_input = EndpointInput(\n",
    "    endpoint_name=endpoint_name, \n",
    "    inference_attribute=\"0\", \n",
    "    destination=\"/opt/ml/processing/input_data\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting existing file\n",
      "Uploading file to s3://sagemaker-us-east-1-343218227212/sagemaker/deepAR-sales-forecasting-ModelQualityMonitor/preprocess.py\n"
     ]
    }
   ],
   "source": [
    "# upload preprocessing script\n",
    "copy_to_s3(\"code/preprocess.py\", f\"s3://{bucket}/{prefix}\" + \"/preprocess.py\", True)\n",
    "preprocessing_script_location = f\"s3://{bucket}/{prefix}\" + \"/preprocess.py\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker.model_monitor import CronExpressionGenerator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:sagemaker.model_monitor.model_monitoring:Creating Monitoring Schedule with name: deepar-sales-forecasting-model-monitoring-schedule-12\n"
     ]
    }
   ],
   "source": [
    "# Create the monitoring schedule to execute every hour.\n",
    "response = deepar_model_quality_monitor.create_monitoring_schedule(\n",
    "    monitor_schedule_name=deepar_monitor_schedule_name,\n",
    "    endpoint_input=endpoint_input,\n",
    "    output_s3_uri=baseline_results_uri,\n",
    "    problem_type=\"Regression\",\n",
    "    #record_preprocessor_script=preprocessing_script_location,\n",
    "    ground_truth_input=ground_truth_upload_path,\n",
    "    constraints=baseline_job.suggested_constraints(),\n",
    "    schedule_cron_expression=CronExpressionGenerator.hourly(),\n",
    "    enable_cloudwatch_metrics=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'MonitoringScheduleArn': 'arn:aws:sagemaker:us-east-1:343218227212:monitoring-schedule/deepar-sales-forecasting-model-monitoring-schedule-12',\n",
       " 'MonitoringScheduleName': 'deepar-sales-forecasting-model-monitoring-schedule-12',\n",
       " 'MonitoringScheduleStatus': 'Scheduled',\n",
       " 'MonitoringType': 'ModelQuality',\n",
       " 'CreationTime': datetime.datetime(2024, 10, 13, 21, 24, 10, 556000, tzinfo=tzlocal()),\n",
       " 'LastModifiedTime': datetime.datetime(2024, 10, 13, 23, 3, 15, 383000, tzinfo=tzlocal()),\n",
       " 'MonitoringScheduleConfig': {'ScheduleConfig': {'ScheduleExpression': 'cron(0 * ? * * *)'},\n",
       "  'MonitoringJobDefinitionName': 'model-quality-job-definition-2024-10-13-21-24-09-864',\n",
       "  'MonitoringType': 'ModelQuality'},\n",
       " 'EndpointName': 'deepar-store-sales-forecasting-model-quality-monitor',\n",
       " 'LastMonitoringExecutionSummary': {'MonitoringScheduleName': 'deepar-sales-forecasting-model-monitoring-schedule-12',\n",
       "  'ScheduledTime': datetime.datetime(2024, 10, 13, 23, 0, tzinfo=tzlocal()),\n",
       "  'CreationTime': datetime.datetime(2024, 10, 13, 23, 1, 27, 438000, tzinfo=tzlocal()),\n",
       "  'LastModifiedTime': datetime.datetime(2024, 10, 13, 23, 3, 15, 373000, tzinfo=tzlocal()),\n",
       "  'MonitoringExecutionStatus': 'Failed',\n",
       "  'EndpointName': 'deepar-store-sales-forecasting-model-quality-monitor',\n",
       "  'FailureReason': 'No S3 objects found under S3 URL \"s3://sagemaker-us-east-1-343218227212/sagemaker/deepAR-sales-forecasting-ModelQualityMonitor/ground_truth_data/2024-10-13-20-30-01/2024/10/13/22\" given in input data source. Please ensure that the bucket exists in the selected region (us-east-1), that objects exist under that S3 prefix, and that the role \"arn:aws:iam::343218227212:role/service-role/AmazonSageMaker-ExecutionRole-20241007T091581\" has \"s3:ListBucket\" permissions on bucket \"sagemaker-us-east-1-343218227212\".'},\n",
       " 'ResponseMetadata': {'RequestId': '7b77b5a2-b828-46d6-9089-9a9728186088',\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HTTPHeaders': {'x-amzn-requestid': '7b77b5a2-b828-46d6-9089-9a9728186088',\n",
       "   'content-type': 'application/x-amz-json-1.1',\n",
       "   'content-length': '1556',\n",
       "   'date': 'Sun, 13 Oct 2024 23:16:51 GMT'},\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# inspect\n",
    "deepar_model_quality_monitor.describe_schedule()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for first execution\n",
      "Execution found!\n"
     ]
    }
   ],
   "source": [
    "# Wait for the first execution of the monitoring_schedule\n",
    "print(\"Waiting for first execution\", end=\"\")\n",
    "while True:\n",
    "    execution = deepar_model_quality_monitor.describe_schedule().get(\n",
    "        \"LastMonitoringExecutionSummary\"\n",
    "    )\n",
    "    if execution:\n",
    "        break\n",
    "    print(\".\", end=\"\", flush=True)\n",
    "    sleep(10)\n",
    "print()\n",
    "print(\"Execution found!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<sagemaker.model_monitor.model_monitoring.MonitoringExecution at 0x7f82a07cd410>]"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get the model monitor executions\n",
    "executions = deepar_model_quality_monitor.list_executions()\n",
    "executions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ProcessingInputs': [{'InputName': 'constraints',\n",
       "   'AppManaged': False,\n",
       "   'S3Input': {'S3Uri': 's3://sagemaker-us-east-1-343218227212/sagemaker/deepAR-sales-forecasting-ModelQualityMonitor/baselining/results/constraints.json',\n",
       "    'LocalPath': '/opt/ml/processing/baseline/constraints',\n",
       "    'S3DataType': 'S3Prefix',\n",
       "    'S3InputMode': 'File',\n",
       "    'S3DataDistributionType': 'FullyReplicated'}},\n",
       "  {'InputName': 'endpoint_input_1',\n",
       "   'AppManaged': False,\n",
       "   'S3Input': {'S3Uri': 's3://sagemaker-us-east-1-343218227212/sagemaker/deepAR-sales-forecasting-ModelQualityMonitor/baselining/results/merge/deepar-store-sales-forecasting-model-quality-monitor/AllTraffic/2024/10/13/21',\n",
       "    'LocalPath': '/opt/ml/processing/input_data/deepar-store-sales-forecasting-model-quality-monitor/AllTraffic/2024/10/13/21',\n",
       "    'S3DataType': 'S3Prefix',\n",
       "    'S3InputMode': 'File',\n",
       "    'S3DataDistributionType': 'FullyReplicated',\n",
       "    'S3CompressionType': 'None'}}],\n",
       " 'ProcessingOutputConfig': {'Outputs': [{'OutputName': 'result',\n",
       "    'S3Output': {'S3Uri': 's3://sagemaker-us-east-1-343218227212/sagemaker/deepAR-sales-forecasting-ModelQualityMonitor/baselining/results/deepar-store-sales-forecasting-model-quality-monitor/deepar-sales-forecasting-model-monitoring-schedule-12/2024/10/13/22',\n",
       "     'LocalPath': '/opt/ml/processing/output',\n",
       "     'S3UploadMode': 'Continuous'},\n",
       "    'AppManaged': False}]},\n",
       " 'ProcessingJobName': 'model-quality-monitoring-202410132200-d8e651aec34a385dc44b256a',\n",
       " 'ProcessingResources': {'ClusterConfig': {'InstanceCount': 1,\n",
       "   'InstanceType': 'ml.m5.2xlarge',\n",
       "   'VolumeSizeInGB': 20}},\n",
       " 'StoppingCondition': {'MaxRuntimeInSeconds': 1800},\n",
       " 'AppSpecification': {'ImageUri': '156813124566.dkr.ecr.us-east-1.amazonaws.com/sagemaker-model-monitor-analyzer'},\n",
       " 'Environment': {'analysis_type': 'MODEL_QUALITY',\n",
       "  'baseline_constraints': '/opt/ml/processing/baseline/constraints/constraints.json',\n",
       "  'dataset_format': '{\"sagemakerMergeJson\":{\"captureIndexNames\":[\"endpointOutput\"],\"originalDatasetFormat\":null}}',\n",
       "  'dataset_source': '/opt/ml/processing/input_data',\n",
       "  'end_time': '2024-10-13T22:00:00Z',\n",
       "  'inference_attribute': '0',\n",
       "  'metric_time': '2024-10-13T21:00:00Z',\n",
       "  'monitoring_input_type': 'ENDPOINT_INPUT',\n",
       "  'output_path': '/opt/ml/processing/output',\n",
       "  'problem_type': 'Regression',\n",
       "  'publish_cloudwatch_metrics': 'Enabled',\n",
       "  'sagemaker_endpoint_name': 'deepar-store-sales-forecasting-model-quality-monitor',\n",
       "  'sagemaker_monitoring_schedule_name': 'deepar-sales-forecasting-model-monitoring-schedule-12',\n",
       "  'start_time': '2024-10-13T21:00:00Z'},\n",
       " 'RoleArn': 'arn:aws:iam::343218227212:role/service-role/AmazonSageMaker-ExecutionRole-20241007T091581',\n",
       " 'ProcessingJobArn': 'arn:aws:sagemaker:us-east-1:343218227212:processing-job/model-quality-monitoring-202410132200-d8e651aec34a385dc44b256a',\n",
       " 'ProcessingJobStatus': 'Completed',\n",
       " 'ExitMessage': 'Completed: Job completed successfully with no violations.',\n",
       " 'ProcessingEndTime': datetime.datetime(2024, 10, 13, 22, 14, 28, 81000, tzinfo=tzlocal()),\n",
       " 'ProcessingStartTime': datetime.datetime(2024, 10, 13, 22, 12, 59, 806000, tzinfo=tzlocal()),\n",
       " 'LastModifiedTime': datetime.datetime(2024, 10, 13, 22, 14, 28, 528000, tzinfo=tzlocal()),\n",
       " 'CreationTime': datetime.datetime(2024, 10, 13, 22, 7, 59, 471000, tzinfo=tzlocal()),\n",
       " 'MonitoringScheduleArn': 'arn:aws:sagemaker:us-east-1:343218227212:monitoring-schedule/deepar-sales-forecasting-model-monitoring-schedule-12',\n",
       " 'ResponseMetadata': {'RequestId': '3b473d2a-8ac0-4b4d-a04a-67ebff4a548e',\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HTTPHeaders': {'x-amzn-requestid': '3b473d2a-8ac0-4b4d-a04a-67ebff4a548e',\n",
       "   'content-type': 'application/x-amz-json-1.1',\n",
       "   'content-length': '3023',\n",
       "   'date': 'Sun, 13 Oct 2024 23:17:16 GMT'},\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get latest executions\n",
    "while not executions:\n",
    "    executions = deepar_model_quality_monitor.list_executions()\n",
    "    print(\".\", end=\"\", flush=True)\n",
    "    sleep(10)\n",
    "latest_execution = executions[-1]\n",
    "latest_execution.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# wait for execution to finish \n",
    "status = execution[\"MonitoringExecutionStatus\"]\n",
    "\n",
    "while status in [\"Pending\", \"InProgress\"]:\n",
    "    print(\"Waiting for execution to finish\", end=\"\")\n",
    "    latest_execution.wait(logs=False)\n",
    "    latest_job = latest_execution.describe()\n",
    "    print()\n",
    "    print(f\"{latest_job['ProcessingJobName']} job status:\", latest_job[\"ProcessingJobStatus\"])\n",
    "    print(\n",
    "        f\"{latest_job['ProcessingJobName']} job exit message, if any:\",\n",
    "        latest_job.get(\"ExitMessage\"),\n",
    "    )\n",
    "    print(\n",
    "        f\"{latest_job['ProcessingJobName']} job failure reason, if any:\",\n",
    "        latest_job.get(\"FailureReason\"),\n",
    "    )\n",
    "    sleep(\n",
    "        30\n",
    "    )  # model quality executions consist of two Processing jobs, wait for second job to start\n",
    "    latest_execution = deepar_model_quality_monitor.list_executions()[-1]\n",
    "    execution = deepar_model_quality_monitor.describe_schedule()[\"LastMonitoringExecutionSummary\"]\n",
    "    status = execution[\"MonitoringExecutionStatus\"]\n",
    "\n",
    "print(\"Execution status is:\", status)\n",
    "\n",
    "if status != \"Completed\":\n",
    "    print(execution)\n",
    "    print(\n",
    "        \"====STOP==== \\n No completed executions to inspect further. Please wait till an execution completes or investigate previously reported failures.\"\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Report Uri: s3://sagemaker-us-east-1-343218227212/sagemaker/deepAR-sales-forecasting-ModelQualityMonitor/baselining/results/deepar-store-sales-forecasting-model-quality-monitor/deepar-sales-forecasting-model-monitoring-schedule-12/2024/10/13/22\n"
     ]
    }
   ],
   "source": [
    "# get link to report\n",
    "latest_execution = deepar_model_quality_monitor.list_executions()[-1]\n",
    "report_uri = latest_execution.describe()[\"ProcessingOutputConfig\"][\"Outputs\"][0][\"S3Output\"][\n",
    "    \"S3Uri\"\n",
    "]\n",
    "print(\"Report Uri:\", report_uri)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cleanup - delete monitor if we don't need anymore\n",
    "DO_CLEANUP = False\n",
    "if(DO_CLEANUP):\n",
    "    deepar_model_quality_monitor.delete_monitoring_schedule()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}