Heterogeneous CW-MDP.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "using POMDPs, POMDPTools, QuickPOMDPs, MCTS, DiscreteValueIteration, POMDPSimulators, POMDPModels, POMDPPolicies\n",
    "using Distributions, Combinatorics, StaticArrays, D3Trees, Random\n",
    "using FileIO, JLD2, TickTock"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "Number_level = 10;\n",
    "#Transition_matrix = [0.86 0.14 0.0 0.0; 0.0 0.83 0.17 0.0;0.0 0.0 0.89 0.11;1.0 0.0 0.0 0.0];\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "reward (generic function with 3 methods)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function reward(s, a,k,s1=10, rng=23)       #s is a vector of number units at each level and a is the number of units we will repair\n",
    "    p=[sum(1-T[i,i,1,j] for i in 1:9) for j in 1:NumberUnits]/sum(sum(1-T[i,i,1,j] for i in 1:9) for j in 1:NumberUnits)   \n",
    "    if s==Number_level\n",
    "            r=failure_penalty +  setup_cost/NumberUnits  + normal_operation  ;\n",
    "        return r\n",
    "       end\n",
    "        r=0\n",
    "     if a==1\n",
    "        if s1>=s\n",
    "            crd = Categorical(T[s,:,1,k]);\n",
    "            s = rand(crd);\n",
    "            r = normal_operation ;\n",
    "         return r\n",
    "        else\n",
    "            return 0.0\n",
    "        end\n",
    "        end\n",
    "    if a==2\n",
    "         if s1>=s\n",
    "            crd = Categorical(T[s,:,1,k]);\n",
    "            s = rand(crd);\n",
    "            r = normal_operation +  setup_cost/NumberUnits ;\n",
    "         return r\n",
    "        else\n",
    "            return 0.0\n",
    "        end\n",
    "        end\n",
    "    if a==3\n",
    "        s1=deepcopy(s)\n",
    "        s=1\n",
    "        r = maintenance_penalty +  setup_cost/NumberUnits \n",
    "         crd = Categorical(T[s,:,1,k]);\n",
    "            s = rand(crd);\n",
    "            r+= normal_operation  ;\n",
    "#            r*=(s1/10);\n",
    "        return r\n",
    "    end\n",
    "    end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "T=zeros(Number_level,Number_level,3,100);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in 1:100\n",
    "    fullname = \"TM_MATRIX/tm\"*string(i);\n",
    "    # generate_transition_matrix(Number_level, fullname) \n",
    "    Transition_matrix = load(fullname*\".jld2\",\"transition_matrix\");\n",
    "    \n",
    "    T[:,:,1,i].=Transition_matrix;\n",
    "    T[:,:,2,i].=Transition_matrix;\n",
    "    for j in 1:10\n",
    "        T[j,:,3,i].=Transition_matrix[1,:];\n",
    "        end\n",
    "    T[10,:,1,i].=Transition_matrix[1,:]\n",
    "T[10,:,2,i].=Transition_matrix[1,:]\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "generative (generic function with 2 methods)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "function generative(s, a,k, rng=23)       #s is a vector of number units at each level and a is the number of units we will repair\n",
    "        if s==Number_level\n",
    "        s=1\n",
    "        crd = Categorical(T[s,:,1,k]);\n",
    "            s = rand(crd);\n",
    "            r=failure_penalty +  setup_cost/NumberUnits + normal_operation  ;\n",
    "        return (sp=s, r=r)\n",
    "       end\n",
    "        r=0\n",
    "     if a==1\n",
    "            crd = Categorical(T[s,:,1,k]);\n",
    "            s = rand(crd);\n",
    "            r = normal_operation ;\n",
    "         return (sp=s, r=r)\n",
    "        end\n",
    "    if a==2\n",
    "            crd = Categorical(T[s,:,1,k]);\n",
    "            s = rand(crd);\n",
    "            r = normal_operation +  setup_cost/NumberUnits ;\n",
    "             return (sp=s, r=r)\n",
    "        end\n",
    "    if a==3\n",
    "\n",
    "        s1=deepcopy(s)\n",
    "        s=1\n",
    "        r = maintenance_penalty +  setup_cost/NumberUnits\n",
    "         crd = Categorical(T[s,:,1,k]);\n",
    "            s = rand(crd);\n",
    "            r+= normal_operation  ;\n",
    "       return (sp=s, r=r)\n",
    "    end\n",
    "    end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ValueIteration (generic function with 1 method)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function ValueIteration(γ)\n",
    "    Q=zeros((Number_level,3,NumberUnits))\n",
    "#     print(Q)\n",
    "    V= rand(Float64, (Number_level,NumberUnits))*1000\n",
    "    diff=0\n",
    "    i=0\n",
    "    \n",
    "    \n",
    "\n",
    "    \n",
    "    \n",
    "    for k in 1:NumberUnits\n",
    "        p=[1 1 1 ]\n",
    "    for i in 1:1500\n",
    "    for s in 1:Number_level\n",
    "        Threads.@threads for a in 1:3\n",
    "                Q[s,a,k]= (reward(s,a,k)) + γ*( sum(T[s,s1,a,k]*(V[s1,k]) for (s1) in 1:Number_level ))\n",
    "        end\n",
    "            p=[exp(lamb*sum(Q[s,1,k])),exp(lamb*(Q[s,2,k])),exp(lamb*(Q[s,3,k]))]\n",
    "            p=p./sum(p)\n",
    "       V[s,k]=findmax(Q[s,1:3,k])[1]\n",
    "\n",
    "        end\n",
    "        \n",
    "        end\n",
    "    end\n",
    "    return Q\n",
    "    end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "decison (generic function with 2 methods)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "\n",
    "function decison(Q, s,n=0.2)\n",
    "   a_final=[]\n",
    "   if Number_level in s\n",
    "        for i in 1:NumberUnits\n",
    "            append!(a_final,argmax(a->Q[s[i],a,i], [2,3]))\n",
    "            end\n",
    "        return a_final\n",
    "    end\n",
    "    q0=0.0\n",
    "    a_0=repeat(1:1,NumberUnits);\n",
    "    for i in 1:NumberUnits\n",
    "            q0+=Q[s[i],1,i]\n",
    "    end\n",
    "    a_final=repeat(1:1,NumberUnits);\n",
    "    b=1\n",
    "    for i in 1:NumberUnits\n",
    "            a_final[i]=argmax(a->Q[s[i],a,i], [2,3])\n",
    "\n",
    "            if a_final[i]==3\n",
    "                b+=1\n",
    "            end\n",
    "    end\n",
    "     \n",
    "        q1=0.0\n",
    "    for i in 1:NumberUnits\n",
    "            q1+=Q[s[i],a_final[i],i]\n",
    "    end\n",
    "    if !(3 in a_final) \n",
    "        a_final=a_0\n",
    "    end\n",
    "    if q0>q1\n",
    "            return a_0\n",
    "        else\n",
    "            return a_final\n",
    "        end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><div style = \"float: left;\"><span>0×8 DataFrame</span></div><div style = \"clear: both;\"></div></div><div class = \"data-frame\" style = \"overflow-x: scroll;\"><table class = \"data-frame\" style = \"margin-bottom: 6px;\"></table></div>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cccccccc}\n",
       "\t& Units & n & s & m & f & lambda & mean & std\\\\\n",
       "\t\\hline\n",
       "\t& Any & Any & Any & Any & Any & Any & Any & Any\\\\\n",
       "\t\\hline\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m0×8 DataFrame\u001b[0m"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "using DataFrames\n",
    "df= DataFrame(Units=[],n=[],s=[],m=[],f=[],lambda=[],mean=[],std=[])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30[0, -800, -200, -1000]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info:  started timer at: 2023-04-17T11:27:59.404\n",
      "└ @ TickTock C:\\Users\\vbansal5\\.julia\\packages\\TickTock\\fGILW\\src\\TickTock.jl:54\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30\n",
      "1\n",
      "-57488.95674916878\n",
      "3001.7002023654836\n",
      "-------------------------\n",
      "40[0, -800, -200, -1000]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info:           46.1415803s: 46 seconds, 141 milliseconds\n",
      "└ @ TickTock C:\\Users\\vbansal5\\.julia\\packages\\TickTock\\fGILW\\src\\TickTock.jl:62\n",
      "┌ Info:  started timer at: 2023-04-17T11:38:38.671\n",
      "└ @ TickTock C:\\Users\\vbansal5\\.julia\\packages\\TickTock\\fGILW\\src\\TickTock.jl:54\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40\n",
      "1\n",
      "-70752.97424441637\n",
      "3262.909098773062\n",
      "-------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info:           58.3381118s: 58 seconds, 338 milliseconds\n",
      "└ @ TickTock C:\\Users\\vbansal5\\.julia\\packages\\TickTock\\fGILW\\src\\TickTock.jl:62\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "# print(Q)\n",
    "cost=[ \n",
    "    \n",
    "    [0 -800 -200 -1000]\n",
    "    ]\n",
    "\n",
    "for units in [30,40]\n",
    "     for c in cost\n",
    "   print(units)  \n",
    "    global normal_operation,setup_cost,maintenance_penalty,failure_penalty=c\n",
    "     println([normal_operation,setup_cost,maintenance_penalty,failure_penalty])   \n",
    "     for lambda in 1\n",
    "        global  lamb=lambda\n",
    "    global NumberUnits=units\n",
    "     Q=ValueIteration(.95)\n",
    "tick()\n",
    "trials=10000;\n",
    "simsteps = 100;\n",
    "results=zeros(trials)*0.1\n",
    " Threads.@threads for h in 1:trials\n",
    "s=repeat(1:1,NumberUnits)\n",
    "r1=[]\n",
    "a1=[]   \n",
    "r=repeat(1:1,NumberUnits)*1.0\n",
    " for k in 1:simsteps\n",
    "    a =decison(Q,s)\n",
    "            r2=0.0\n",
    "    for i in 1:NumberUnits\n",
    "         s[i],r[i]=generative(s[i],a[i],i,234);\n",
    "          r2+=r[i]\n",
    "            end \n",
    "    s=s;\n",
    "    append!(a1,a)\n",
    "    append!(r1,r2)\n",
    "end\n",
    "rk=0\n",
    "for i in 1:simsteps\n",
    "    rk=0.95*rk+r1[simsteps-i+1]\n",
    "end\n",
    "    results[h]=rk\n",
    "end\n",
    "println(units)\n",
    "println(lambda)\n",
    "\n",
    "println(mean(results))\n",
    "\n",
    "println(std(results))\n",
    "pushfirst!(df,[units,normal_operation,setup_cost,maintenance_penalty,failure_penalty,lamb,mean(results),std(results)])\n",
    "        \n",
    "tock()\n",
    "end\n",
    "    println(\"-------------------------\")\n",
    "end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><div style = \"float: left;\"><span>2×8 DataFrame</span></div><div style = \"clear: both;\"></div></div><div class = \"data-frame\" style = \"overflow-x: scroll;\"><table class = \"data-frame\" style = \"margin-bottom: 6px;\"><thead><tr class = \"header\"><th class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">Row</th><th style = \"text-align: left;\">Units</th><th style = \"text-align: left;\">n</th><th style = \"text-align: left;\">s</th><th style = \"text-align: left;\">m</th><th style = \"text-align: left;\">f</th><th style = \"text-align: left;\">lambda</th><th style = \"text-align: left;\">mean</th><th style = \"text-align: left;\">std</th></tr><tr class = \"subheader headerLastRow\"><th class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\"></th><th title = \"Any\" style = \"text-align: left;\">Any</th><th title = \"Any\" style = \"text-align: left;\">Any</th><th title = \"Any\" style = \"text-align: left;\">Any</th><th title = \"Any\" style = \"text-align: left;\">Any</th><th title = \"Any\" style = \"text-align: left;\">Any</th><th title = \"Any\" style = \"text-align: left;\">Any</th><th title = \"Any\" style = \"text-align: left;\">Any</th><th title = \"Any\" style = \"text-align: left;\">Any</th></tr></thead><tbody><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">1</td><td style = \"text-align: left;\">40.0</td><td style = \"text-align: left;\">0.0</td><td style = \"text-align: left;\">-800.0</td><td style = \"text-align: left;\">-200.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">1.0</td><td style = \"text-align: left;\">-70753.0</td><td style = \"text-align: left;\">3262.91</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">2</td><td style = \"text-align: left;\">30.0</td><td style = \"text-align: left;\">0.0</td><td style = \"text-align: left;\">-800.0</td><td style = \"text-align: left;\">-200.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">1.0</td><td style = \"text-align: left;\">-57489.0</td><td style = \"text-align: left;\">3001.7</td></tr></tbody></table></div>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cccccccc}\n",
       "\t& Units & n & s & m & f & lambda & mean & std\\\\\n",
       "\t\\hline\n",
       "\t& Any & Any & Any & Any & Any & Any & Any & Any\\\\\n",
       "\t\\hline\n",
       "\t1 & 40.0 & 0.0 & -800.0 & -200.0 & -1000.0 & 1.0 & -70753.0 & 3262.91 \\\\\n",
       "\t2 & 30.0 & 0.0 & -800.0 & -200.0 & -1000.0 & 1.0 & -57489.0 & 3001.7 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m2×8 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m Units \u001b[0m\u001b[1m n   \u001b[0m\u001b[1m s      \u001b[0m\u001b[1m m      \u001b[0m\u001b[1m f       \u001b[0m\u001b[1m lambda \u001b[0m\u001b[1m mean     \u001b[0m\u001b[1m std     \u001b[0m\n",
       "     │\u001b[90m Any   \u001b[0m\u001b[90m Any \u001b[0m\u001b[90m Any    \u001b[0m\u001b[90m Any    \u001b[0m\u001b[90m Any     \u001b[0m\u001b[90m Any    \u001b[0m\u001b[90m Any      \u001b[0m\u001b[90m Any     \u001b[0m\n",
       "─────┼────────────────────────────────────────────────────────────────\n",
       "   1 │ 40.0   0.0  -800.0  -200.0  -1000.0  1.0     -70753.0  3262.91\n",
       "   2 │ 30.0   0.0  -800.0  -200.0  -1000.0  1.0     -57489.0  3001.7"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"CW-MDP_HETERO.csv\""
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "using CSV\n",
    "CSV.write(\"CW-MDP_HETERO.csv\", df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.8.2",
   "language": "julia",
   "name": "julia-1.8"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}