Resource_Recommendation_Algorithm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preliminaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Constants"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Converstion constants to convert CPU in Nanocores to cores and Memory in bytes to MiB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CPU_CONSTANT = 1e9\n",
    "MEM_CONSTANT = 1.048576e+6"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "source_data_date = \"25-04-2024\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CPU & Memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_main_cpu = pd.read_csv(f'./data/raw/cpu/cpu-{source_data_date}.csv')\n",
    "df_main_memory = pd.read_csv(f'./data/raw/memory/memory-{source_data_date}.csv')\n",
    "\n",
    "#for both df_main_cpu and df_main_memory make the timeGenerated to pandas and component to the component name\n",
    "df_main_cpu['TimeGenerated'] = pd.to_datetime(df_main_cpu['TimeGenerated'])\n",
    "df_main_cpu['Component'] = df_main_cpu['Name'].str.split('-').str[:-3].str.join('-')\n",
    "df_main_cpu['CpuCounter'] /= CPU_CONSTANT\n",
    "\n",
    "df_main_memory['TimeGenerated'] = pd.to_datetime(df_main_memory['TimeGenerated'])\n",
    "df_main_memory['Component'] = df_main_memory['Name'].str.split('-').str[:-3].str.join('-')\n",
    "df_main_memory['MemoryCounter'] /= MEM_CONSTANT"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Instances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_instances = pd.read_csv(f'./data/raw/instances/instances-{source_data_date}.csv')\n",
    "\n",
    "def clean_data(df_instances):\n",
    "    \"\"\"\n",
    "    Cleans and transforms the instances data.\n",
    "\n",
    "    This function performs the following steps:\n",
    "    1. Reshapes the data using a pivot table to have Names as rows and CounterNames as columns.\n",
    "    2. Fills missing values with 0.\n",
    "    3. Drops duplicate rows.\n",
    "    4. Converts CPU and memory values to more readable units (cores and MiB).\n",
    "    5. Renames columns for clarity.\n",
    "\n",
    "    Args:\n",
    "    df_instances: The DataFrame containing the instances data.\n",
    "\n",
    "    Returns:\n",
    "    A cleaned and transformed DataFrame.\n",
    "    \"\"\"\n",
    "\n",
    "    # Reshape the data using pivot_table to have Names as rows and CounterNames as columns\n",
    "    # Filling missing values with 0 to avoid errors in calculations\n",
    "    df_instances = df_instances.pivot_table(index=['Name'], columns='CounterName', values='Value', fill_value=0) #fills missing values with 0\n",
    "    df_instances = df_instances.reset_index()\n",
    "    df_instances.drop_duplicates()\n",
    "    df_instances[\"cpuLimitNanoCores\"] /= CPU_CONSTANT\n",
    "    df_instances[\"cpuRequestNanoCores\"] /=CPU_CONSTANT\n",
    "    df_instances[\"memoryLimitBytes\"] /=MEM_CONSTANT\n",
    "    df_instances[\"memoryRequestBytes\"] /=MEM_CONSTANT\n",
    "\n",
    "    df_instances = df_instances.rename(columns={'cpuLimitNanoCores': 'cpuLimitCores','cpuRequestNanoCores': 'cpuRequestCores','memoryLimitBytes': 'memoryLimitMiB','memoryRequestBytes': 'memoryRequestMiB'})\n",
    "    return df_instances\n",
    "\n",
    "df_instances = clean_data(df_instances.copy())\n",
    "df_instances"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## HPA Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_hpa = pd.read_csv(f'./data/raw/instances/hpa-{source_data_date}.csv')\n",
    "\n",
    "def clean_data(df_hpa):\n",
    "    \"\"\"\n",
    "    Cleans and transforms the Horziontal Pod Autoscaling data.\n",
    "\n",
    "    This function performs the following steps:\n",
    "        1. Filters rows based on the 'ns' column to include only a specific namespace.\n",
    "        2. Drops unnecessary columns.\n",
    "        3. Drops duplicate rows.\n",
    "        4. Fills missing values in 'last_scale_time' with 0 and converts non-zero values to 1, This is to denote which components underwent scaling in that period.\n",
    "        5. Extracts relevant information from 'deployment_hpa' and 'ns' columns.\n",
    "        6. Renames columns for clarity.\n",
    "\n",
    "    Args:\n",
    "        df_hpa: The DataFrame containing the HPA data.\n",
    "\n",
    "    Returns:\n",
    "        A cleaned and transformed DataFrame.\n",
    "    \"\"\"\n",
    "\n",
    "    # Filter rows based on column: 'ns'\n",
    "    df_hpa = df_hpa[df_hpa['ns'].str.startswith(\"staging\", na=False)]\n",
    "    # Drop column: 'Cluster'\n",
    "    df_hpa = df_hpa.drop(columns=['Cluster', 'deployment_hpa2','scale_out_percentage','Computer', 'Origin', 'Namespace', 'Name',\n",
    "    'AgentId', 'Tags', 'TimeGenerated [Sri Jayawardenepura]', 'Type', '_ResourceId', 'pTags', 'Val'])\n",
    "    # Drop duplicate rows across all columns\n",
    "    df_hpa = df_hpa.drop_duplicates()\n",
    "    # if there is a missing value for last_scale_time fill it with zero, make the others 1\n",
    "    df_hpa['last_scale_time'] = df_hpa['last_scale_time'].fillna(0)\n",
    "    #values other than zero are converted to 1\n",
    "    df_hpa['last_scale_time'] = df_hpa['last_scale_time'].apply(lambda x: 1 if x != 0 else 0)\n",
    "    df_hpa['deployment_hpa'] = df_hpa['deployment_hpa'].str.split('-').str[:-1].str.join('-')\n",
    "    df_hpa['ns'] = df_hpa['ns'].str.split('-').str[2:-2].str.join('-')\n",
    "    # Change the column name to last_scaled\n",
    "    df_hpa = df_hpa.rename(columns={'last_scale_time': 'LastScaled', 'ns': 'Namespace', 'deployment_hpa': 'Component', 'desired_reps': 'DesiredReplicas', 'min_reps': 'MinReplicas', 'max_reps': 'MaxReplicas'})\n",
    "    return df_hpa\n",
    "\n",
    "df_hpa = clean_data(df_hpa.copy())\n",
    "df_hpa.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculate Pod Duration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pod_duration(df_main_cpu):\n",
    "    df_pod_duration = df_main_cpu.groupby('Name').agg({'TimeGenerated': ['min', 'max'], 'Component': 'first'})\n",
    "    df_pod_duration['TimeDifference'] = df_pod_duration['TimeGenerated']['max'] - df_pod_duration['TimeGenerated']['min']\n",
    "    df_pod_duration.reset_index(inplace=True)\n",
    "    df_pod_duration.columns = ['Name', 'TimeStamp_min', 'TimeStamp_max', 'Component','TimeDifference']\n",
    "    return df_pod_duration\n",
    "\n",
    "df_pod_duration = pod_duration(df_main_cpu.copy())\n",
    "df_pod_duration.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check Unique Components"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # make it into a dataframe\n",
    "# unique_components = pd.DataFrame(df_pod_duration['Component'].unique(), columns=['Component'])\n",
    "# #make this single column into a single row\n",
    "# unique_components = unique_components.transpose()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Grouped Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_merged = df_instances.merge(df_pod_duration, on='Name', how='left')\n",
    "df_merged = df_merged.merge(df_hpa, on='Component', how='left')\n",
    "df_merged = df_merged.dropna(subset=['TimeDifference', 'Component'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Inner Join CPU and Memory Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Inner Join (Note: Inner and Outerjoin gave same results)\n",
    "df_main_inner = df_main_cpu.merge(df_main_memory, on=['TimeGenerated', 'Name'], how='inner', suffixes=('', '_memory'))\n",
    "df_main_inner = df_main_inner.merge(df_merged, on='Name', how='inner', suffixes=('', '_instances'))\n",
    "\n",
    "def clean_data(df_main_inner):\n",
    "    # Drop columns: 'Component_memory', 'PodStartTime_memory' and 3 other columns\n",
    "    df_main_inner = df_main_inner.drop(columns=['Component_memory', 'PodStartTime_memory', 'Namespace_memory', 'PodCreationTimeStamp_memory', 'InstanceName_memory', 'Namespace_instances', 'Component_instances'])\n",
    "    return df_main_inner\n",
    "\n",
    "df_main_inner = clean_data(df_main_inner.copy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Recommendations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is the optimized version of the recommendation algorithm."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimized_recommender(data, instances,  window='7d', offset='0d', algorithmCPU='Quantile', algorithmRAM='Quantile', qCPU=0.9999, qRAM=0.95,  targetCPUUtilization=0.7, targetRAMUtilization=0.8, minRecCpuCores=0.1, minRecRamMiB=20, cpu_limit_factor=2, mem_limit_factor=2):\n",
    "    \"\"\"This function takes resource utilization data (`data`) and deployment configuration data (`instances`) and generates recommendations for resource requests and limits for each deployment in `instances`.\n",
    "\n",
    "    Args:\n",
    "      data: A pandas DataFrame containing resource utilization data. This DataFrame is expected to have the CpuCounter, MemoryCounter and 'TimeGenerated' columns.\n",
    "      instances: A pandas DataFrame containing deployment configuration data. This DataFrame is expected to have columns named 'Component', 'Name', 'MinReplicas', 'MaxReplicas', 'cpuRequestCores', 'cpuLimitCores', 'memoryRequestMiB', 'memoryLimitMiB', and 'TimeDifference'.\n",
    "      window: A string representing the time window to consider for calculating resource utilization. Defaults to '3d' (3 days).\n",
    "      offset: A string representing the offset to apply to the most recent time in the data before considering the window. Defaults to '0d' (no offset). This can be used for shifting the time window.\n",
    "      algorithmCPU: A string indicating the algorithm to use for calculating CPU utilization. Can be 'Max' (maximum) or 'Quantile' (quantile). Defaults to 'Quantile'.\n",
    "      algorithmRAM: A string indicating the algorithm to use for calculating RAM utilization. Can be 'Max' (maximum) or 'Quantile' (quantile). Defaults to 'Quantile'.\n",
    "      qCPU: A float between 0 and 1 representing the quantile to use for calculating CPU utilization if algorithmCPU is set to 'Quantile'. Defaults to 0.9999 (99.99th percentile).\n",
    "      qRAM: A float between 0 and 1 representing the quantile to use for calculating RAM utilization if algorithmRAM is set to 'Quantile'. Defaults to 0.95 (95th percentile).\n",
    "      targetCPUUtilization: A float between 0 and 1 representing the target CPU utilization for deployments. This is an additional buffer provided. Defaults to 0.7 (70%).\n",
    "      targetRAMUtilization: A float between 0 and 1 representing the target RAM utilization for deployments. This is an additional buffer provided. Defaults to 0.8 (80%).\n",
    "      minRecCpuCores: A float representing the minimum recommended CPU request cores. Defaults to 0.1.\n",
    "      minRecRamMiB: An integer representing the minimum recommended RAM request in MiB. Defaults to 20.\n",
    "      cpu_limit_factor: A float representing the factor to apply to the recommended CPU request to calculate the recommended CPU limit. Defaults to 2.\n",
    "      mem_limit_factor: A float representing the factor to apply to the recommended RAM request to calculate the recommended RAM limit. Defaults to 2.\n",
    "\n",
    "    Returns:\n",
    "      A pandas DataFrame containing the updated deployment configuration data with recommended CPU and RAM requests and limits.\n",
    "    \"\"\"\n",
    "   \n",
    "    # Ensure 'TimeGenerated' is in datetime format\n",
    "    data['TimeGenerated'] = pd.to_datetime(data['TimeGenerated'])\n",
    "    instances['TimeDifference'] = pd.to_timedelta(instances['TimeDifference'])   \n",
    "    \n",
    "    # Copy and filter instances data to ensure we don't modify the original DataFrame\n",
    "    instances_data = instances.copy()\n",
    "    instances_data = instances_data.dropna(subset=['TimeDifference', 'Component', 'MinReplicas'])\n",
    "    \n",
    "    # Define lambda functions for CPU and RAM calculations based on chosen algorithm\n",
    "    cpu_func = lambda x: x.max() if algorithmCPU == 'Max' else x.quantile(qCPU)\n",
    "    ram_func = lambda x: x.max() if algorithmRAM == 'Max' else x.quantile(qRAM)\n",
    "    \n",
    "    # Get unique components and iterate through them\n",
    "    components = instances['Component'].unique()\n",
    "    updates = []\n",
    "    for component in components:\n",
    "        # Filter data based on component\n",
    "        component_data = instances_data[instances_data['Component'] == component]\n",
    "        # Filter data based on window criteria\n",
    "        component_data = component_data[component_data['TimeDifference'] >= pd.Timedelta(window)]\n",
    "        if component_data.empty:\n",
    "            continue\n",
    "          \n",
    "        # Filter main data for the current component and time window\n",
    "        main_data = data[data['Component'] == component]\n",
    "        #get the most recent time in the data - offset\n",
    "        end_time = main_data['TimeGenerated'].max()- pd.Timedelta(offset)\n",
    "        #limit the data to the window\n",
    "        main_data = main_data[(main_data['TimeGenerated'] >= end_time - pd.Timedelta(window)) & (main_data['TimeGenerated'] <= end_time)]\n",
    "\n",
    "        # Group data by 'Name' for each component to reduce computations\n",
    "        grouped_data = main_data.groupby('Name').agg(\n",
    "            base_cpu_recommendation=('CpuCounter', cpu_func),\n",
    "            base_mem_recommendation=('MemoryCounter', ram_func)\n",
    "        ).reset_index()\n",
    "        \n",
    "        # Apply calculations for recommended requests and limits\n",
    "        grouped_data['RecommendedCpuRequestCores'] = grouped_data['base_cpu_recommendation'].apply(lambda x: max(round(x / targetCPUUtilization, 3), minRecCpuCores))\n",
    "        grouped_data['RecommendedMemoryRequestMiB'] = grouped_data['base_mem_recommendation'].apply(lambda x: max(round(x / targetRAMUtilization, 0), minRecRamMiB))\n",
    "        grouped_data['RecommendedCpuLimitCores'] = grouped_data['RecommendedCpuRequestCores'].apply(lambda x: round(x * cpu_limit_factor, 3))\n",
    "        grouped_data['RecommendedMemoryLimitMiB'] = grouped_data['RecommendedMemoryRequestMiB'].apply(lambda x: round(x * mem_limit_factor, 0))\n",
    "        \n",
    "        #concatenate the grouped data\n",
    "        updates.append(grouped_data[['Name', 'RecommendedCpuRequestCores', 'RecommendedMemoryRequestMiB', 'RecommendedCpuLimitCores', 'RecommendedMemoryLimitMiB']])\n",
    "        \n",
    "    updates_df = pd.concat(updates, ignore_index=True)\n",
    "    instances_data = pd.merge(instances, updates_df, on='Name', how='left')\n",
    "    \n",
    "    # Group by component and aggregate final resource utilization data\n",
    "    instances_data = instances_data.groupby('Component').agg(\n",
    "        Namespace=('Namespace', 'first'),\n",
    "        cpuRequestCores=('cpuRequestCores', 'max'),\n",
    "        cpuLimitCores=('cpuLimitCores', 'max'),\n",
    "        memoryRequestMiB=('memoryRequestMiB', 'max'),\n",
    "        memoryLimitMiB=('memoryLimitMiB', 'max'),\n",
    "        MinReplicas=('MinReplicas', 'max'),\n",
    "        MaxReplicas=('MaxReplicas', 'max'),\n",
    "        RecommendedCpuRequestCores=('RecommendedCpuRequestCores', 'max'),\n",
    "        RecommendedCpuLimitCores=('RecommendedCpuLimitCores', 'max'),\n",
    "        RecommendedMemoryRequestMiB=('RecommendedMemoryRequestMiB', 'max'),\n",
    "        RecommendedMemoryLimitMiB=('RecommendedMemoryLimitMiB', 'max')\n",
    "    ).reset_index()\n",
    "    instances_data = instances_data.dropna(subset=['RecommendedCpuRequestCores', 'RecommendedCpuLimitCores', 'RecommendedMemoryRequestMiB', 'RecommendedMemoryLimitMiB'])\n",
    "    return instances_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculate Recommendations\n",
    "\n",
    "This is used to caculate recommendations with different parameters and different windows. The '7d' window and '0d' offset is what is mostly used."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "windows = ['3d', '7d']\n",
    "offsets = ['0d', '1d', '2d']\n",
    "\n",
    "for window in windows:\n",
    "    for offset in offsets:\n",
    "        df_recommended = optimized_recommender(df_main_inner, df_merged, window=window, offset=offset, algorithmCPU='Max', algorithmRAM='Quantile', qCPU=0.9999, qRAM=0.95,  targetCPUUtilization=0.7, targetRAMUtilization=0.8, minRecCpuCores=0.1, minRecRamMiB=20, cpu_limit_factor=2, mem_limit_factor=2)\n",
    "        df_recommended.to_csv(f'./reports/Recommendations-{source_data_date}-{window}-{offset}Offset.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "data-analysis",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}