Skip to content

Commit

Permalink
Refactor get_abs_tol and get_rel_tol functions to remove unnecessary …
Browse files Browse the repository at this point in the history 
…code
  • Loading branch information
@perceptualrobots
perceptualrobots committed Sep 8, 2024
1 parent 9bf147f commit cc060ba
Show file tree
Hide file tree
Showing 9 changed files with 349 additions and 85 deletions.
4 changes: 3 additions & 1 deletion nbs/01_putils.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -934,6 +934,7 @@
"def get_abs_tol(key):\n",
" # dic = {'evolve': 0.01, 'ARC-evolve' : 0.01, 'ARC-display': 0.01, 'ARC': 0.01}\n",
" dic = { 'ARC-evolve' : 0.01, 'ARC-display': 0.1, 'ARC-change' : 0.01, 'ARC-zero': 0.01, 'ARC-gradient': 0.0001}\n",
" dic = {}\n",
"\n",
" # if key in dic:\n",
" return dic[key]\n",
Expand All @@ -956,7 +957,8 @@
"source": [
"#| export\n",
"def get_rel_tol(key):\n",
" dic = { 'ARC-change' : 1e-3}\n",
" # dic = { 'ARC-change' : 1e-3}\n",
" dic = { }\n",
"\n",
" # if key in dic:\n",
" return dic[key]\n",
Expand Down
14 changes: 7 additions & 7 deletions nbs/05_environments.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1557,22 +1557,22 @@
" return self.value\n",
"\n",
" def set_properties(self, props: dict) -> None:\n",
"\n",
" # data = props['data']\n",
"\n",
" data_mgr = DataManagerSingleton.get_instance(folder = props['dir'], prefix = props['file_prefix'], show_timing=True)\n",
" data = data_mgr.get_data_for_code(props['code'])\n",
" props['test_output_array'] = data_mgr.get_solutions_for_code(props['code'])\n",
"\n",
" self.env.initialise(props, data)\n",
" self.fitness = self.env.fitness\n",
" self.history = props.get('history', 5)\n",
" self.initial = props.get('initial', 1000)\n",
" self.grid_shape = props.get('grid_shape', None)\n",
" self.input_set = props.get('input_set', None)\n",
" # self.boxcar = [self.initial for i in range(1, self.history+1)]\n",
" self.reset_boxcar()\n",
" self.num_links = self.env.get_num_actions()\n",
" self.rel_tol_ARC_change = props['tolerances']['rtARCchange']\n",
" self.abs_tol_ARC_change = props['tolerances']['atARCchange']\n",
" self.abs_tol_ARC_gradient = props['tolerances']['atARCgradient']\n",
" self.abs_tol_ARC_zero = props['tolerances']['atARCzero']\n",
"\n",
"\n",
" def get_properties(self):\n",
" env_inputs_names = self.get_env_inputs_names()\n",
Expand Down Expand Up @@ -1667,7 +1667,7 @@
" if self.env_done:\n",
" self.done = self.env_done\n",
" else:\n",
" self.done, details = ListChecker.check_list_unchanged(self.boxcar, rel_tol =get_rel_tol('ARC-change'), abs_tol=get_abs_tol('ARC-change'), gradient_abs_tol=get_abs_tol('ARC-gradient'))\n",
" self.done, details = ListChecker.check_list_unchanged(self.boxcar, rel_tol =self.rel_tol_ARC_change, abs_tol=self.abs_tol_ARC_change, gradient_abs_tol=self.abs_tol_ARC_gradient)\n",
" if self.done:\n",
" self.env.add_to_gradient_list(details['gradient_range']) \n",
"\n",
Expand All @@ -1693,7 +1693,7 @@
"\n",
" def is_fitness_close_to_zero(self):\n",
" # should this be max of fitness list\n",
" return ListChecker.check_float_list_close_to_zero(self.boxcar, rel_tol = 0, abs_tol=get_abs_tol('ARC-zero'))\n",
" return ListChecker.check_float_list_close_to_zero(self.boxcar, rel_tol = 0, abs_tol=self.abs_tol_ARC_zero)\n",
"\n",
"\n",
" def get_fitness_list(self):\n",
Expand Down
232 changes: 194 additions & 38 deletions nbs/15_arc.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@
"import os, math\n",
"import numpy as np\n",
"import gym\n",
"from time import sleep\n"
"from time import sleep\n",
"from abc import ABC, abstractmethod"
]
},
{
Expand All @@ -55,6 +56,153 @@
"from pct.helpers import ChallengesDataManager"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# ARC Fitness"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Factory"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"class ARCFitnessFactory:\n",
" factories = {}\n",
" def addFactory(id, functionFactory):\n",
" ARCFitnessFactory.factories.put[id] = functionFactory\n",
" addFactory = staticmethod(addFactory)\n",
" # A Template Method:\n",
" def createFunction(id):\n",
" if not ARCFitnessFactory.factories.__contains__(id):\n",
" ARCFitnessFactory.factories[id] = \\\n",
" eval(id + '.Factory()')\n",
" return ARCFitnessFactory.factories[id].create()\n",
" createFunction = staticmethod(createFunction)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## BaseARCFitness"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"class BaseARCFitness(ABC):\n",
" \"Base class of an ARCFitness. This class is not used directly by developers, but defines the functionality common to all.\"\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## SumSquareOfDiff"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"class SumSquareOfDiff(BaseARCFitness):\n",
" \"A function that sums the square of the diffrences of two arrays.\"\n",
" \n",
" def __call__(self, output_array=None, env_array=None, control_set=None):\n",
" # First metric: square of the difference between the dimensions\n",
" dim_metric = 0\n",
" if 'dims' in control_set and len(control_set) == 1:\n",
" dim_metric = (env_array.shape[0] - output_array.shape[0]) ** 2 + (env_array.shape[1] - output_array.shape[1]) ** 2\n",
"\n",
" # Second metric: square of the difference between each element in the arrays\n",
" element_metric = 0\n",
" if 'cells' in control_set:\n",
" diff = env_array[:output_array.shape[0], :output_array.shape[1]] - output_array\n",
" diff = np.where(np.isnan(diff), 0, diff)\n",
" diff = np.where(np.isnan(env_array[:output_array.shape[0], :output_array.shape[1]]), 10, diff)\n",
" element_metric = np.sum(diff ** 2)\n",
"\n",
" # Final metric\n",
" if 'dims' in control_set and len(control_set) == 1:\n",
" final_metric = dim_metric\n",
" elif 'cells' in control_set and len(control_set) == 1:\n",
" final_metric = element_metric\n",
" else:\n",
" final_metric = dim_metric + element_metric\n",
"\n",
" return final_metric\n",
"\n",
" class Factory:\n",
" def create(self): return SumSquareOfDiff() \n",
" "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## AverageMaxOfDiff"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"class AverageMaxOfDiff(BaseARCFitness):\n",
" \"A function that sums the square of the diffrences of two arrays.\"\n",
" \n",
" def __call__(self, output_array=None, env_array=None, control_set=None):\n",
" # First metric: square of the difference between the dimensions\n",
" dim_metric = 0\n",
" if 'dims' in control_set and len(control_set) == 1:\n",
" dim_metric = (env_array.shape[0] - output_array.shape[0]) ** 2 + (env_array.shape[1] - output_array.shape[1]) ** 2\n",
"\n",
" # Second metric: square of the difference between each element in the arrays\n",
" element_metric = 0\n",
" if 'cells' in control_set:\n",
" diff = env_array[:output_array.shape[0], :output_array.shape[1]] - output_array\n",
" diff = np.where(np.isnan(diff), 0, diff)\n",
" diff = np.where(np.isnan(env_array[:output_array.shape[0], :output_array.shape[1]]), 10, diff)\n",
" abs_diff = np.abs(diff)\n",
" maximum = np.max(abs_diff)\n",
" average = np.average(abs_diff)\n",
" element_metric = max(average, maximum)\n",
"\n",
" # Final metric\n",
" if 'dims' in control_set and len(control_set) == 1:\n",
" final_metric = dim_metric\n",
" elif 'cells' in control_set and len(control_set) == 1:\n",
" final_metric = element_metric\n",
" else:\n",
" final_metric = dim_metric + element_metric\n",
"\n",
" return final_metric\n",
"\n",
" class Factory:\n",
" def create(self): return AverageMaxOfDiff() "
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand All @@ -71,6 +219,9 @@
"#| export\n",
"class ARCDataProcessor:\n",
" def __init__(self, config_dict, arc_dict):\n",
"\n",
" self.fitness_function_type = ARCFitnessFactory.createFunction(config_dict.get('fitness', 'SumSquareOfDiff'))\n",
"\n",
" self.arc_dict = arc_dict\n",
" self.grid_shape = self.determine_grid_shape()\n",
" \n",
Expand Down Expand Up @@ -258,28 +409,33 @@
" return np.array(self.arc_dict[dataset][self.index]['output'])\n",
"\n",
" def fitness_function_arrays(self, output_array, env_array):\n",
" # First metric: square of the difference between the dimensions\n",
" dim_metric = 0\n",
" if 'dims' in self.control_set and len(self.control_set) == 1:\n",
" dim_metric = (env_array.shape[0] - output_array.shape[0]) ** 2 + (env_array.shape[1] - output_array.shape[1]) ** 2\n",
"\n",
" # Second metric: square of the difference between each element in the arrays\n",
" element_metric = 0\n",
" if 'cells' in self.control_set:\n",
" diff = env_array[:output_array.shape[0], :output_array.shape[1]] - output_array\n",
" diff = np.where(np.isnan(diff), 0, diff)\n",
" diff = np.where(np.isnan(env_array[:output_array.shape[0], :output_array.shape[1]]), 10, diff)\n",
" element_metric = np.sum(diff ** 2)\n",
"\n",
" # Final metric\n",
" if 'dims' in self.control_set and len(self.control_set) == 1:\n",
" final_metric = dim_metric\n",
" elif 'cells' in self.control_set and len(self.control_set) == 1:\n",
" final_metric = element_metric\n",
" else:\n",
" final_metric = dim_metric + element_metric\n",
" return self.fitness_function_type(output_array = output_array, env_array = env_array, control_set = self.control_set) \n",
"\n",
" return final_metric\n",
"\n",
" # # First metric: square of the difference between the dimensions\n",
" # dim_metric = 0\n",
" # if 'dims' in self.control_set and len(self.control_set) == 1:\n",
" # dim_metric = (env_array.shape[0] - output_array.shape[0]) ** 2 + (env_array.shape[1] - output_array.shape[1]) ** 2\n",
"\n",
" # # Second metric: square of the difference between each element in the arrays\n",
" # element_metric = 0\n",
" # if 'cells' in self.control_set:\n",
" # diff = env_array[:output_array.shape[0], :output_array.shape[1]] - output_array\n",
" # diff = np.where(np.isnan(diff), 0, diff)\n",
" # diff = np.where(np.isnan(env_array[:output_array.shape[0], :output_array.shape[1]]), 10, diff)\n",
" # element_metric = np.sum(diff ** 2)\n",
"\n",
" # # Final metric\n",
" # if 'dims' in self.control_set and len(self.control_set) == 1:\n",
" # final_metric = dim_metric\n",
" # elif 'cells' in self.control_set and len(self.control_set) == 1:\n",
" # final_metric = element_metric\n",
" # else:\n",
" # final_metric = dim_metric + element_metric\n",
"\n",
" # return final_metric\n",
"\n",
" def fitness_function(self):\n",
" output_array = self.get_output(self.dataset)\n",
Expand Down Expand Up @@ -487,23 +643,21 @@
"{'num_actions': 9, 'env': (3, 3)}\n",
"names ['IE001', 'IE002', 'IE003', 'IE004', 'IE005', 'IE006', 'IE007', 'IE008', 'IE009']\n",
"indexes [0, 1, 2, 3, 4, 5, 6, 7, 8]\n",
"{'num_actions': 9, 'env': (3, 3)}\n",
"fitness 2143.0 {'inputs': {'cells': {'env': array([[0., 7., 7.],\n",
" [7., 7., 7.],\n",
" [0., 7., 7.]], dtype=float32)}}}\n",
"\n",
"[-2, -2, -2, -2, -2, -2, -2, -2, -2]\n",
"{'num_actions': 9, 'env': (3, 3)}\n",
"fitness 1975.0 {'inputs': {'cells': {'env': array([[0., 5., 5.],\n",
" [5., 5., 5.],\n",
" [0., 5., 5.]], dtype=float32)}}}\n",
"\n",
"[-3, -3, -3, -3, -3, -3, -3, -3, -3]\n",
"{'num_actions': 9, 'env': (3, 3)}\n",
"fitness 1828.0 {'inputs': {'cells': {'env': array([[0., 2., 2.],\n",
" [2., 2., 2.],\n",
" [0., 2., 2.]], dtype=float32)}}}\n",
"\n"
"{'num_actions': 9, 'env': (3, 3)}\n"
]
},
{
"ename": "ValueError",
"evalue": "operands could not be broadcast together with shapes (3,3) (9,9) ",
"output_type": "error",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[1;32mIn[22], line 21\u001b[0m\n\u001b[0;32m 19\u001b[0m state, info \u001b[38;5;241m=\u001b[39m gp\u001b[38;5;241m.\u001b[39mget_state()\n\u001b[0;32m 20\u001b[0m \u001b[38;5;28mprint\u001b[39m(info)\n\u001b[1;32m---> 21\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfitness\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[43mgp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfitness_function\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m, state)\n\u001b[0;32m 22\u001b[0m \u001b[38;5;28mprint\u001b[39m()\n\u001b[0;32m 24\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m2\u001b[39m,\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m4\u001b[39m,\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m):\n",
"Cell \u001b[1;32mIn[20], line 225\u001b[0m, in \u001b[0;36mARCDataProcessor.fitness_function\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 223\u001b[0m output_array \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_output(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset)\n\u001b[0;32m 224\u001b[0m env_array \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39menv\n\u001b[1;32m--> 225\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfitness_function_arrays\u001b[49m\u001b[43m(\u001b[49m\u001b[43moutput_array\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43menv_array\u001b[49m\u001b[43m)\u001b[49m\n",
"Cell \u001b[1;32mIn[20], line 196\u001b[0m, in \u001b[0;36mARCDataProcessor.fitness_function_arrays\u001b[1;34m(self, output_array, env_array)\u001b[0m\n\u001b[0;32m 193\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfitness_function_arrays\u001b[39m(\u001b[38;5;28mself\u001b[39m, output_array, env_array):\n\u001b[1;32m--> 196\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfitness_function_type\u001b[49m\u001b[43m(\u001b[49m\u001b[43moutput_array\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43moutput_array\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43menv_array\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43menv_array\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcontrol_set\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcontrol_set\u001b[49m\u001b[43m)\u001b[49m\n",
"Cell \u001b[1;32mIn[18], line 14\u001b[0m, in \u001b[0;36mSumSquareOfDiff.__call__\u001b[1;34m(self, output_array, env_array, control_set)\u001b[0m\n\u001b[0;32m 12\u001b[0m element_metric \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[0;32m 13\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcells\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;129;01min\u001b[39;00m control_set:\n\u001b[1;32m---> 14\u001b[0m diff \u001b[38;5;241m=\u001b[39m \u001b[43menv_array\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[43moutput_array\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mshape\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m:\u001b[49m\u001b[43moutput_array\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mshape\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m]\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[43m \u001b[49m\u001b[43moutput_array\u001b[49m\n\u001b[0;32m 15\u001b[0m diff \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mwhere(np\u001b[38;5;241m.\u001b[39misnan(diff), \u001b[38;5;241m0\u001b[39m, diff)\n\u001b[0;32m 16\u001b[0m diff \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mwhere(np\u001b[38;5;241m.\u001b[39misnan(env_array[:output_array\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m0\u001b[39m], :output_array\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m1\u001b[39m]]), \u001b[38;5;241m10\u001b[39m, diff)\n",
"\u001b[1;31mValueError\u001b[0m: operands could not be broadcast together with shapes (3,3) (9,9) "
]
}
],
Expand Down Expand Up @@ -656,6 +810,8 @@
" self.code = properties.get('code', \"\")\n",
" self.arc_data = ARCDataProcessor(properties, arc_dict)\n",
" self.runs = properties.get('runs', len(arc_dict['train'])) if 'index' in properties else properties.get('runs', 1) / len(arc_dict['train'])\n",
" self.abs_tol_ARC_evolve = properties['tolerances']['atARCevolve']\n",
"\n",
" self.reset()\n",
"\n",
" def step(self, actions):\n",
Expand Down Expand Up @@ -726,7 +882,7 @@
"\n",
" def is_environment_resolved(self):\n",
" max_fitness = max(self.fitness_list)\n",
" return max_fitness < 0.01\n",
" return max_fitness < self.abs_tol_ARC_evolve \n",
"\n",
"\n",
" def get_environment_score(self):\n",
Expand Down
5 changes: 2 additions & 3 deletions nbs/16_environment_processing.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -554,8 +554,7 @@
" # ,enhanced_environment_properties=enhanced_environment_properties\n",
" )\n",
"\n",
" score = round(score ** 0.5, 1)\n",
" print('Test score',score)\n",
" print(f'Test score = {score:4.3f}')\n",
" fitness_list = str( [f'{i:4.3f}' for i in self.env_processing_details['fitness_list']])\n",
" print('fitness_list', fitness_list)\n",
"\n",
Expand All @@ -582,7 +581,7 @@
" experiment.log_other('input_set', str(input_set))\n",
" experiment.log_metric('last_gen', self.env_processing_details['last_gen'])\n",
" experiment.log_metric('fitness', self.env_processing_details['fitness'])\n",
" experiment.log_other('test_score', f'{score:4.3f}')\n",
" experiment.log_metric('test_score', round(score, 3))\n",
" experiment.log_other('code', environment_properties['code'])\n",
" success = self.success(gradient_list, self.env_processing_details['fitness'], score) \n",
" experiment.log_metric('success', success)\n",
Expand Down
Loading

0 comments on commit cc060ba

Please sign in to comment.