Homogeneous CW-MDP.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "using POMDPs, POMDPTools, QuickPOMDPs, MCTS, DiscreteValueIteration, POMDPSimulators, POMDPModels, POMDPPolicies\n",
    "using Distributions, Combinatorics, StaticArrays, D3Trees, Random\n",
    "using FileIO, JLD2, TickTock"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "Number_level = 10;\n",
    "#Transition_matrix = [0.86 0.14 0.0 0.0; 0.0 0.83 0.17 0.0;0.0 0.0 0.89 0.11;1.0 0.0 0.0 0.0];\n",
    "fullname = \"tm10\";\n",
    "\n",
    "Transition_matrix = load(fullname*\".jld2\",\"transition_matrix\");\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "reward (generic function with 3 methods)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function reward(s, a,s1=10, rng=23)       #s is a vector of number units at each level and a is the number of units we will repair\n",
    "       if s==Number_level\n",
    "            r=failure_penalty +  setup_cost/NumberUnits  + normal_operation  ;\n",
    "        return r\n",
    "       end\n",
    "        r=0\n",
    "     if a==1\n",
    "        if s1>=s\n",
    "            crd = Categorical(Transition_matrix[s,:]);\n",
    "            s = rand(crd);\n",
    "            r = normal_operation ;\n",
    "         return r\n",
    "        else\n",
    "            return 0.0\n",
    "        end\n",
    "        end\n",
    "    if a==2\n",
    "         if s1>=s\n",
    "            crd = Categorical(Transition_matrix[s,:]);\n",
    "            s = rand(crd);\n",
    "            r = normal_operation +  setup_cost/NumberUnits;\n",
    "         return r\n",
    "        else\n",
    "            return 0.0\n",
    "        end\n",
    "        end\n",
    "    if a==3\n",
    "        s1=deepcopy(s)\n",
    "        s=1\n",
    "        r = maintenance_penalty +  setup_cost/NumberUnits\n",
    "         crd = Categorical(Transition_matrix[s,:]);\n",
    "            s = rand(crd);\n",
    "            r+= normal_operation  ;\n",
    "#            r*=(s1/10);\n",
    "        return r\n",
    "    end\n",
    "    end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10-element view(::Array{Float64, 3}, 10, :, 2) with eltype Float64:\n",
       " 0.2695279572108137\n",
       " 0.16860741623354286\n",
       " 0.1306613209439592\n",
       " 0.09857168678543618\n",
       " 0.09171796107138079\n",
       " 0.09134806335671229\n",
       " 0.09048875946531719\n",
       " 0.04832973229444604\n",
       " 0.00607725632113932\n",
       " 0.004669846317252394"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T=zeros(Number_level,Number_level,3);\n",
    "\n",
    "\n",
    "T[:,:,1].=Transition_matrix;\n",
    "\n",
    "T[:,:,2].=Transition_matrix;\n",
    "\n",
    "for j in 1:10\n",
    "        T[j,:,3].=Transition_matrix[1,:];\n",
    "        end\n",
    "\n",
    "T[10,:,1].=Transition_matrix[1,:]\n",
    "T[10,:,2].=Transition_matrix[1,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "generative (generic function with 2 methods)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "function generative(s, a, rng=23)       #s is state of a component and a is extend action of a component\n",
    "        if s==Number_level\n",
    "        s=1\n",
    "        crd = Categorical(Transition_matrix[s,:]);\n",
    "            s = rand(crd);\n",
    "            r=failure_penalty +  setup_cost/NumberUnits + normal_operation  ;\n",
    "        return (sp=s, r=r)\n",
    "       end\n",
    "        r=0\n",
    "     if a==1\n",
    "            crd = Categorical(Transition_matrix[s,:]);\n",
    "            s = rand(crd);\n",
    "            r = normal_operation ;\n",
    "         return (sp=s, r=r)\n",
    "        end\n",
    "    if a==2\n",
    "            crd = Categorical(Transition_matrix[s,:]);\n",
    "            s = rand(crd);\n",
    "            r = normal_operation +  setup_cost/NumberUnits ;\n",
    "             return (sp=s, r=r)\n",
    "        end\n",
    "    if a==3\n",
    "        s1=deepcopy(s)\n",
    "        s=1\n",
    "        r = maintenance_penalty +  setup_cost/NumberUnits\n",
    "         crd = Categorical(Transition_matrix[s,:]);\n",
    "            s = rand(crd);\n",
    "            r+= normal_operation  ;\n",
    "       return (sp=s, r=r)\n",
    "    end\n",
    "    end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ValueIteration (generic function with 1 method)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function ValueIteration(γ)\n",
    "    Q=zeros((Number_level,3))\n",
    "    V= rand(Float64, (Number_level))*1000\n",
    "    i=0\n",
    "    while i<=2000\n",
    "    for s in 1:10\n",
    "        Threads.@threads for a in 1:3\n",
    " \n",
    "                Q[s,a]= (reward(s,a)) + γ*( sum(T[s,s1,a]*(V[s1]) for (s1) in 1:10 ))\n",
    "    end\n",
    "            V[s]=findmax(Q[s,1:3])[1]\n",
    "        end\n",
    "        i+=1\n",
    "        end\n",
    "    return Q,V\n",
    "    end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "decison (generic function with 1 method)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "function decison(Q, s)\n",
    "   a_final=[]\n",
    "   if Number_level in s\n",
    "        for i in 1:NumberUnits\n",
    "            append!(a_final,argmax(a->Q[s[i],a], [2,3]))\n",
    "            end\n",
    "        return a_final\n",
    "    end\n",
    "    q0=0.0\n",
    "    a_0=repeat(1:1,NumberUnits);\n",
    "    for i in 1:NumberUnits\n",
    "            q0+=Q[s[i],1]\n",
    "    end\n",
    "    a_final=repeat(1:1,NumberUnits);\n",
    "    repair= argmax(i->s[i], 1:NumberUnits)\n",
    "    a_final[repair]=3\n",
    "    b=1\n",
    "    for i in 1:NumberUnits\n",
    "            if i!=repair\n",
    "            a_final[i]=argmax(a->Q[s[i],a], [2,3])\n",
    "            if a_final[i]==3\n",
    "                b+=1\n",
    "            end\n",
    "            end\n",
    "    end\n",
    "        q1=0.0\n",
    "    for i in 1:NumberUnits\n",
    "            q1+=Q[s[i],a_final[i]]\n",
    "    end\n",
    "    if q0>q1\n",
    "            return a_0\n",
    "        else\n",
    "            return a_final\n",
    "        end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><div style = \"float: left;\"><span>0×8 DataFrame</span></div><div style = \"clear: both;\"></div></div><div class = \"data-frame\" style = \"overflow-x: scroll;\"><table class = \"data-frame\" style = \"margin-bottom: 6px;\"></table></div>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cccccccc}\n",
       "\t& Units & n & s & m & f & lambda & mean & std\\\\\n",
       "\t\\hline\n",
       "\t& Any & Any & Any & Any & Any & Any & Any & Any\\\\\n",
       "\t\\hline\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m0×8 DataFrame\u001b[0m"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "using DataFrames\n",
    "df= DataFrame(Units=[],n=[],s=[],m=[],f=[],lambda=[],mean=[],std=[])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20[0, -1000, -200, -1000]\n",
      "false\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info:  started timer at: 2023-04-17T10:17:44.419\n",
      "└ @ TickTock C:\\Users\\vbansal5\\.julia\\packages\\TickTock\\fGILW\\src\\TickTock.jl:54\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20\n",
      "1\n",
      "-36018.46858660268\n",
      "2156.139617029731\n",
      "-------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info:           13.1088172s: 13 seconds, 108 milliseconds\n",
      "└ @ TickTock C:\\Users\\vbansal5\\.julia\\packages\\TickTock\\fGILW\\src\\TickTock.jl:62\n"
     ]
    }
   ],
   "source": [
    "\n",
    "cost=[ \n",
    "     [0 -1000 -200 -1000]\n",
    "    ]\n",
    "\n",
    "\n",
    "for units in [20]\n",
    "    for c in cost\n",
    "   print(units)  \n",
    "    global normal_operation,setup_cost,maintenance_penalty,failure_penalty=c\n",
    "     println([normal_operation,setup_cost,maintenance_penalty,failure_penalty])   \n",
    "     for lambda in 1:1:1\n",
    "        global  lamb=lambda\n",
    "    global NumberUnits=units\n",
    "     Q,V=ValueIteration(.95)\n",
    "         println(isnan(Q[1,1]))\n",
    "       if sum(isnan(Q[1,1]))>0.0\n",
    "            continue\n",
    "        end\n",
    "tick()\n",
    "trials=10000;\n",
    "simsteps = 100;\n",
    "results=zeros(trials)*0.1\n",
    "Threads.@threads for h in 1:trials\n",
    "s=repeat(1:1,NumberUnits)\n",
    "r1=[]\n",
    "a1=[]   \n",
    "r=repeat(1:1,NumberUnits)*1.0\n",
    " for k in 1:simsteps\n",
    "    a =decison(Q,s)\n",
    "            r2=0.0\n",
    "    for i in 1:NumberUnits\n",
    "         s[i],r[i]=generative(s[i],a[i],234);\n",
    "          r2+=r[i]\n",
    "            end \n",
    "    s=s;\n",
    "    append!(a1,a)\n",
    "    append!(r1,r2)\n",
    "end\n",
    "rk=0\n",
    "for i in 1:simsteps\n",
    "    rk=0.95*rk+r1[simsteps-i+1]\n",
    "end\n",
    "    results[h]=rk\n",
    "end\n",
    "println(units)\n",
    "println(lambda)\n",
    "\n",
    "println(mean(results))\n",
    "\n",
    "println(std(results))\n",
    "pushfirst!(df,[units,normal_operation,setup_cost,maintenance_penalty,failure_penalty,lamb,mean(results),std(results)])\n",
    "        \n",
    "tock()\n",
    "end\n",
    "    println(\"-------------------------\")\n",
    "end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><div style = \"float: left;\"><span>1×8 DataFrame</span></div><div style = \"clear: both;\"></div></div><div class = \"data-frame\" style = \"overflow-x: scroll;\"><table class = \"data-frame\" style = \"margin-bottom: 6px;\"><thead><tr class = \"header\"><th class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">Row</th><th style = \"text-align: left;\">Units</th><th style = \"text-align: left;\">n</th><th style = \"text-align: left;\">s</th><th style = \"text-align: left;\">m</th><th style = \"text-align: left;\">f</th><th style = \"text-align: left;\">lambda</th><th style = \"text-align: left;\">mean</th><th style = \"text-align: left;\">std</th></tr><tr class = \"subheader headerLastRow\"><th class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\"></th><th title = \"Any\" style = \"text-align: left;\">Any</th><th title = \"Any\" style = \"text-align: left;\">Any</th><th title = \"Any\" style = \"text-align: left;\">Any</th><th title = \"Any\" style = \"text-align: left;\">Any</th><th title = \"Any\" style = \"text-align: left;\">Any</th><th title = \"Any\" style = \"text-align: left;\">Any</th><th title = \"Any\" style = \"text-align: left;\">Any</th><th title = \"Any\" style = \"text-align: left;\">Any</th></tr></thead><tbody><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">1</td><td style = \"text-align: left;\">20.0</td><td style = \"text-align: left;\">0.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">-200.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">1.0</td><td style = \"text-align: left;\">-36018.5</td><td style = \"text-align: left;\">2156.14</td></tr></tbody></table></div>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cccccccc}\n",
       "\t& Units & n & s & m & f & lambda & mean & std\\\\\n",
       "\t\\hline\n",
       "\t& Any & Any & Any & Any & Any & Any & Any & Any\\\\\n",
       "\t\\hline\n",
       "\t1 & 20.0 & 0.0 & -1000.0 & -200.0 & -1000.0 & 1.0 & -36018.5 & 2156.14 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m1×8 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m Units \u001b[0m\u001b[1m n   \u001b[0m\u001b[1m s       \u001b[0m\u001b[1m m      \u001b[0m\u001b[1m f       \u001b[0m\u001b[1m lambda \u001b[0m\u001b[1m mean     \u001b[0m\u001b[1m std     \u001b[0m\n",
       "     │\u001b[90m Any   \u001b[0m\u001b[90m Any \u001b[0m\u001b[90m Any     \u001b[0m\u001b[90m Any    \u001b[0m\u001b[90m Any     \u001b[0m\u001b[90m Any    \u001b[0m\u001b[90m Any      \u001b[0m\u001b[90m Any     \u001b[0m\n",
       "─────┼─────────────────────────────────────────────────────────────────\n",
       "   1 │ 20.0   0.0  -1000.0  -200.0  -1000.0  1.0     -36018.5  2156.14"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"CW-MDP.csv\""
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "using CSV\n",
    "CSV.write(\"CW-MDP.csv\", df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.8.2",
   "language": "julia",
   "name": "julia-1.8"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}