Merge pull request #51 from NREL/fix_num_dims

Fix num dims

.gitignore

@@ -3,4 +3,5 @@
 MANIFEST
 dist
 build
-.pytest_cache
+.pytest_cache
+gdxpds/test/pytest.log

gdxpds/gdx.py

@@ -628,8 +628,8 @@ def __init__(self,name,data_type,dims=0,file=None,index=None,
         self._data_type = GamsDataType(data_type)
         self._variable_type = None; self.variable_type = variable_type
         self._equation_type = None; self.equation_type = equation_type
-        self._dataframe = None
-        self._dims = []; self.dims = dims       
+        self._dataframe = None; self._dims = None
+        self.dims = dims       
         assert self._dataframe is not None
         self._file = file
         self._index = index        
@@ -770,7 +770,7 @@ def get_value_col_default(self,value_col_name):
         value_col = GamsValueType(value_col_name)
         if self.data_type == GamsDataType.Set:
             assert value_col == GamsValueType.Level
-            return True
+            return c_bool(True)
         if (self.data_type == GamsDataType.Variable) and (
                (value_col == GamsValueType.Lower) or 
                (value_col == GamsValueType.Upper)):
@@ -808,10 +808,10 @@ def dims(self):
 
     @dims.setter
     def dims(self, value):
-        if self.loaded and self.num_records > 0:
+        if (self._dims is not None) and (self.loaded and ((self.num_dims > 0) or (self.num_records > 0))):
             if not isinstance(value,list) or len(value) != self.num_dims:
-                logger.warning("Cannot set dims to {}, because dataframe with dims {} already contains data.".format(value,self.dims))
-        if isinstance(value,int):
+                raise Error("Cannot set dims to {}, because the number of dimensions has already been set to {}.".format(value,self.num_dims))
+        if isinstance(value, int):
             self._dims = ['*'] * value
             self._init_dataframe()
             return
@@ -820,8 +820,7 @@ def dims(self, value):
         for dim in value:
             if not isinstance(dim,string_types):
                 raise Error('Individual dimensions must be denoted by strings. Was passed {} as element of {}.'.format(dim, value))
-        if self.num_dims > 0 and self.num_dims != len(value):
-            logger.warning("{}'s number of dimensions is changing from {} to {}.".format(self.name,self.num_dims,len(value)))
+        assert (self._dims is None) or (self.loaded and (self.num_dims == 0) and (self.num_records == 0)) or (len(value) == self.num_dims)
         self._dims = value
         if self.loaded and self.num_records > 0:
             self._dataframe.columns = self.dims + self.value_col_names
@@ -838,26 +837,62 @@ def dataframe(self):
 
     @dataframe.setter
     def dataframe(self, data):
-        try:
+        try:        
+            # get data in common format and start dealing with dimensions    
             if isinstance(data, pds.DataFrame):
-                # Fix up dimensions
-                num_dims = len(data.columns) - len(self.value_cols)
-                dim_cols = list(data.columns[:num_dims])
-                #logger.debug("When setting dataframe for {}, found {} dimensions with columns labeled {}.".format(self.name,num_dims,dim_cols))
-                replace_dims = True
-                for col in dim_cols:
-                    if not isinstance(col,string_types):
-                        replace_dims = False
-                        logger.info("Not using dataframe column names to set dimensions because {} is not a string.".format(col))
-                        break
-                if replace_dims:
-                    self.dims = dim_cols
-                if num_dims != self.num_dims:
-                    self.dims = num_dims
-                self._dataframe = copy.deepcopy(data)
-                self._dataframe.columns = self.dims + self.value_col_names
+                df = copy.deepcopy(data)
+                has_col_names = True
             else:
-                self._dataframe = pds.DataFrame(data,columns=self.dims + self.value_col_names)
+                df = pds.DataFrame(data)
+                has_col_names = False
+                if df.empty:
+                    # clarify dimensionality, as needed for loading empty GdxSymbols
+                    df = pds.DataFrame(data,columns=self.dims + self.value_cols)
+
+            # finish handling dimensions
+            n = len(df.columns)
+            if (self.num_dims > 0) or (self.num_records > 0):
+                if not ((n == self.num_dims) or (n == self.num_dims + len(self.value_cols))):
+                    raise Error("Cannot set dataframe to {} because the number ".format(df.head()) + \
+                        "of dimensions would change. This symbol has {} ".format(self.num_dims) + \
+                        "dimensions, currently represented by {}.".format(self.dims))
+                num_dims = self.num_dims
+            else:
+                # num_dims not explicitly established yet. in this case we must 
+                # assume value columns have been provided or dimensionality is 0
+                num_dims = max(n - len(self.value_cols),0)
+                if (num_dims == 0) and (n < len(self.value_cols)):
+                    raise Error("Cannot set dataframe to {} because the number ".format(df.head()) + \
+                        "of dimensions cannot be established consistent with {}.".format(self))
+                if self.loaded and (num_dims > 0):
+                    logger.warning('Inferring {} to have {} dimensions. '.format(self.name,num_dims) + 
+                        'Recommended practice is to explicitly set gdxpds.gdx.GdxSymbol dims in the constructor.')
+
+            replace_dims = True
+            if has_col_names:
+                dim_cols = list(df.columns)[:num_dims]
+            elif self.num_dims == num_dims:
+                dim_cols = self.dims
+                replace_dims = False
+            else:
+                dim_cols = ['*'] * num_dims            
+            for col in dim_cols:
+                if not isinstance(col,string_types):
+                    replace_dims = False
+                    logger.info("Not using dataframe column names to set dimensions because {} is not a string.".format(col))
+                    if num_dims != self.num_dims:
+                        self.dims = num_dims
+                    break
+            if replace_dims:
+                self.dims = dim_cols
+            # all done establishing dimensions
+            assert self.num_dims == num_dims
+
+            # finalize the dataframe
+            if n == self.num_dims:
+                self._append_default_values(df)
+            df.columns = self.dims + self.value_col_names
+            self._dataframe = df
         except Exception:
             logger.error("Unable to set dataframe for {} to\n{}\n\nIn process dataframe: {}".format(self,data,self._dataframe))
             raise
@@ -874,6 +909,12 @@ def _init_dataframe(self):
             replace_df_column(self._dataframe,colname,self._dataframe[colname].astype(c_bool))
         return
 
+    def _append_default_values(self,df):
+        assert len(df.columns) == self.num_dims
+        logger.debug("Applying default values to create valid dataframe for '{}'.".format(self.name))
+        for value_col_name in self.value_col_names:
+            df[value_col_name] = self.get_value_col_default(value_col_name)
+
     def _fixup_set_value(self):
         """
         Tricky to get boolean set values to come through right. 

gdxpds/test/test_details.py

@@ -53,6 +53,7 @@
 import gdxcc
 import numpy as np
 import pandas as pds
+import pytest
 
 logger = logging.getLogger(__name__)
 
@@ -236,3 +237,236 @@ def test_unnamed_dimensions(manage_rundir):
         assert gdx['star_eqn'].data_type == gdxpds.gdx.GamsDataType.Equation
         assert gdx['star_eqn'].variable_type is None
         assert gdx['star_eqn'].equation_type == gdxpds.gdx.GamsEquationType.GreaterThan
+
+def test_setting_dataframes(manage_rundir):
+    outdir = os.path.join(run_dir,'setting_dataframes')
+    if not os.path.exists(outdir):
+        os.mkdir(outdir)
+
+    with gdxpds.gdx.GdxFile() as gdx:
+        # reading is tested elsewhere. here go through the different ways to
+        # set a dataframe. 
+
+        # start with WAYS THAT WORK:
+        # 0 dims
+        #     full dataframe
+        gdx.append(gdxpds.gdx.GdxSymbol('sym_1',gdxpds.gdx.GamsDataType.Parameter))
+        gdx[-1].dataframe = pds.DataFrame([[2.0]])
+        assert list(gdx[-1].dataframe.columns) == ['Value']
+        #     edit initialized dataframe - Parameter
+        gdx.append(gdxpds.gdx.GdxSymbol('sym_2',gdxpds.gdx.GamsDataType.Parameter))
+        n = len(gdx[-1].dataframe.columns)
+        gdx[-1].dataframe['Value'] = [5.0] # list is required to specify number of rows to make
+        assert n == len(gdx[-1].dataframe.columns)
+        #     list of lists
+        gdx.append(gdxpds.gdx.GdxSymbol('sym_3',gdxpds.gdx.GamsDataType.Variable))
+        values = [3.0]
+        for value_col_name in gdx[-1].value_col_names:
+            if value_col_name == 'Level':
+                continue
+            values.append(gdx[-1].get_value_col_default(value_col_name))
+        gdx[-1].dataframe = [values]
+        #     reset with empty list
+        gdx.append(gdxpds.gdx.GdxSymbol('sym_4',gdxpds.gdx.GamsDataType.Parameter))
+        gdx[-1].dataframe = pds.DataFrame([[1.0]])
+        gdx[-1].dataframe = []
+        assert gdx[-1].num_records == 0
+
+        # > 0 dims - GdxSymbol initialized with dims=0
+        #     full dataframe
+        gdx.append(gdxpds.gdx.GdxSymbol('sym_5',gdxpds.gdx.GamsDataType.Parameter))
+        gdx[-1].dataframe = pds.DataFrame([['u1','CC',8727.2],
+                                           ['u2','CC',7500.2],
+                                           ['u3','CT',9258.0]], 
+                                          columns=['u','q','val'])
+        assert gdx[-1].num_dims == 2
+        assert gdx[-1].num_records == 3
+        #     full list of lists
+        gdx.append(gdxpds.gdx.GdxSymbol('sym_6',gdxpds.gdx.GamsDataType.Parameter))
+        gdx[-1].dataframe = [['u1','CC',8727.2],
+                             ['u2','CC',7500.2],
+                             ['u3','CT',9258.0],
+                             ['u4','Coal',10100.0]]
+        assert gdx[-1].num_dims == 2
+        assert gdx[-1].num_records == 4
+        #     reset with empty list
+        gdx.append(gdxpds.gdx.GdxSymbol('sym_7',gdxpds.gdx.GamsDataType.Parameter))
+        gdx[-1].dataframe = gdx[-2].dataframe.copy()
+        gdx[-1].dataframe = []
+        assert gdx[-1].num_dims == 2
+        assert gdx[-1].num_records == 0
+
+        # > 0 dims - GdxSymbol initialized with dims=n
+        #     dataframe of dims
+        gdx.append(gdxpds.gdx.GdxSymbol('sym_8',gdxpds.gdx.GamsDataType.Variable,
+            dims=3,variable_type=gdxpds.gdx.GamsVariableType.Positive))
+        gdx[-1].dataframe = pds.DataFrame([['u0','BES','c2'],
+                                           ['u0','BES','c1'],
+                                           ['u1','BES','c2']])
+        assert gdx[-1].num_dims == 3;
+        assert gdx[-1].dims == ['*','*','*']
+        assert len(gdx[-1].dataframe.columns) > 3
+        gdx[-1].dataframe[gdxpds.gdx.GamsValueType.Level.name] = 1.0
+        gdx[-1].dataframe[gdxpds.gdx.GamsValueType.Upper.name] = 10.0
+        #     full dataframe
+        gdx.append(gdxpds.gdx.GdxSymbol('sym_9',gdxpds.gdx.GamsDataType.Parameter,
+            dims=3))
+        gdx[-1].dataframe = pds.DataFrame([['u0','BES','c2',2.0],
+                                           ['u0','BES','c1',1.0],
+                                           ['u1','BES','c2',2.0]],
+                                          columns=['u','q','c','storage_duration_h'])
+        assert list(gdx[-1].dataframe.columns) == ['u','q','c','Value']
+        #     list of lists containing dims only
+        gdx.append(gdxpds.gdx.GdxSymbol('sym_10',gdxpds.gdx.GamsDataType.Equation,
+            dims=4,equation_type=gdxpds.gdx.GamsEquationType.LessThan))
+        gdx[-1].dataframe = [['u0','PHES','c0','1'],
+                             ['u0','PHES','c0','2'],
+                             ['u0','PHES','c0','3'],
+                             ['u0','PHES','c0','4'],
+                             ['u0','PHES','c0','5']]
+        gdx[-1].dataframe['Level'] = -15.0
+        assert list(gdx[-1].dataframe.columns[:gdx[-1].num_dims]) == ['*'] * 4
+        #     full list of lists
+        gdx.append(gdxpds.gdx.GdxSymbol('sym_11',gdxpds.gdx.GamsDataType.Set,
+            dims=2))
+        gdx[-1].dataframe = [['PV','c0',True],
+                             ['CSP','c0',False],
+                             ['CSP','c1',False],
+                             ['Wind','c0',True]]
+        assert gdx[-1].num_dims == 2
+        #     reset with empty list
+        gdx.append(gdxpds.gdx.GdxSymbol('sym_12',gdxpds.gdx.GamsDataType.Set,
+            dims=2))
+        gdx[-1].dataframe = gdx[-1].dataframe.copy()
+        gdx[-1].dataframe = []
+        assert gdx[-1].num_dims == 2
+        assert gdx[-1].dims == ['*'] * 2
+        assert gdx[-1].num_records == 0
+
+        # > 0 dims - GdxSymbol initialized with dims=[list of actual dims]
+        #     dataframe of dims
+        gdx.append(gdxpds.gdx.GdxSymbol('sym_13',gdxpds.gdx.GamsDataType.Variable,
+            dims=['u','q','c'],variable_type=gdxpds.gdx.GamsVariableType.Positive))
+        gdx[-1].dataframe = pds.DataFrame([['u0','BES','c2'],
+                                           ['u0','BES','c1'],
+                                           ['u1','BES','c2']])
+        assert gdx[-1].num_dims == 3;
+        assert gdx[-1].dims == ['u','q','c']
+        assert len(gdx[-1].dataframe.columns) > 3
+        gdx[-1].dataframe[gdxpds.gdx.GamsValueType.Level.name] = 1.0
+        gdx[-1].dataframe[gdxpds.gdx.GamsValueType.Upper.name] = 10.0
+        #     full dataframe
+        gdx.append(gdxpds.gdx.GdxSymbol('sym_14',gdxpds.gdx.GamsDataType.Parameter,
+            dims=['u','q','c']))
+        gdx[-1].dataframe = pds.DataFrame([['u0','BES','c2',2.0],
+                                           ['u0','BES','c1',1.0],
+                                           ['u1','BES','c2',2.0]],
+                                          columns=['u','q','c','storage_duration_h'])
+        assert list(gdx[-1].dataframe.columns) == ['u','q','c','Value']
+        #     list of lists containing dims only
+        gdx.append(gdxpds.gdx.GdxSymbol('sym_15',gdxpds.gdx.GamsDataType.Equation,
+            dims=['u','q','c','t'],equation_type=gdxpds.gdx.GamsEquationType.LessThan))
+        gdx[-1].dataframe = [['u0','PHES','c0','1'],
+                             ['u0','PHES','c0','2'],
+                             ['u0','PHES','c0','3'],
+                             ['u0','PHES','c0','4'],
+                             ['u0','PHES','c0','5']]
+        gdx[-1].dataframe['Level'] = -15.0
+        assert list(gdx[-1].dataframe.columns[:gdx[-1].num_dims]) == ['u','q','c','t']
+        #     full list of lists
+        gdx.append(gdxpds.gdx.GdxSymbol('sym_16',gdxpds.gdx.GamsDataType.Set,
+            dims=['q','c']))
+        gdx[-1].dataframe = [['PV','c0',True],
+                             ['CSP','c0',False],
+                             ['CSP','c1',False],
+                             ['Wind','c0',True]]
+        assert gdx[-1].num_dims == 2
+        #     reset with empty list
+        gdx.append(gdxpds.gdx.GdxSymbol('sym_17',gdxpds.gdx.GamsDataType.Set,
+            dims=['q','c']))
+        gdx[-1].dataframe = gdx['sym_11'].dataframe.copy()
+        gdx[-1].dataframe = []
+        assert gdx[-1].num_dims == 2
+        assert list(gdx[-1].dataframe.columns[:gdx[-1].num_dims]) == ['*'] * 2
+        assert gdx[-1].num_records == 0
+
+        # And then document that some ways DO NOT WORK:
+        # dims=0
+        #     set value, then try to set different number of dimensions
+        gdx.append(gdxpds.gdx.GdxSymbol('sym_18',gdxpds.gdx.GamsDataType.Parameter,
+            dims=0))
+        gdx[-1].dataframe = [[3]]
+        with pytest.raises(Exception) as e_info:
+            gdx[-1].dims = 3
+        # dims > 0
+        #     explicitly set dims to something else
+        gdx.append(gdxpds.gdx.GdxSymbol('sym_19',gdxpds.gdx.GamsDataType.Parameter,
+                   dims=['g','t']))
+        with pytest.raises(Exception) as e_info:
+            gdx[-1].dims = ['g','t','d']
+        #     dataframe of different number of dims
+        gdx.append(gdxpds.gdx.GdxSymbol('sym_20',gdxpds.gdx.GamsDataType.Variable,
+            dims=['d','t']))
+        gdx[-1].dataframe = [['d1','1'],
+                             ['d1','2'],
+                             ['d1','3']]
+        tmp = gdx[-1].dataframe.copy()
+        cols = list(tmp.columns)
+        tmp['q'] = 'PV'
+        tmp = tmp[['q'] + cols]
+        with pytest.raises(Exception) as e_info:
+            gdx[-1].dataframe = tmp
+        #     full dataframe of different number of dims
+        gdx.append(gdxpds.gdx.GdxSymbol('sym_21',gdxpds.gdx.GamsDataType.Parameter,
+            dims=6))
+        assert gdx[-1].dims == ['*'] * 6
+        with pytest.raises(Exception) as e_info:
+            gdx[-1].dataframe = pds.DataFrame([['1',6.0],
+                                              ['2',7.0],
+                                              ['3',-12.0]])
+        #     list of lists of varying widths
+        gdx.append(gdxpds.gdx.GdxSymbol('sym_22',gdxpds.gdx.GamsDataType.Parameter,
+            dims=3))
+        with pytest.raises(Exception) as e_info:
+            gdx[-1].dataframe = [[1]]
+        with pytest.raises(Exception) as e_info:
+            gdx[-1].dataframe = [['1',2.5],
+                                 ['2',-30.0]]
+        # TODO: Write test where parameter value ends up as set dimension--does
+        # an exception get thrown upon writing to GDX?
+        with pytest.raises(Exception) as e_info:
+            gdx[-1].dataframe = [['u1','PV','c0','1',2.5],
+                                 ['u1','PV','c0','2',-30.0]]
+
+        gdx.write(os.path.join(outdir,'dataframe_set_tests.gdx'))
+    with gdxpds.gdx.GdxFile(lazy_load=False) as gdx:
+        gdx.read(os.path.join(outdir,'dataframe_set_tests.gdx'))
+        assert gdx['sym_1'].num_records == 1
+        assert gdx['sym_2'].num_records == 1
+        assert gdx['sym_3'].num_records == 1
+        assert gdx['sym_4'].num_records == 1 # GAMS defaults empty 0-dim parameter to 0
+        assert gdx['sym_4'].dataframe['Value'].values[0] == 0.0
+        assert gdx['sym_5'].dims == ['u','q']
+        assert gdx['sym_5'].num_records == 3
+        assert gdx['sym_5'].dataframe['Value'].values[1] == 7500.2
+        assert gdx['sym_6'].num_records == 4
+        assert gdx['sym_7'].num_records == 0
+        assert gdx['sym_8'].dims == ['*'] * 3
+        assert gdx['sym_8'].dataframe[gdxpds.gdx.GamsValueType.Upper.name].values[0] == 10.0
+        assert gdx['sym_9'].num_records == 3
+        assert gdx['sym_10'].num_dims == 4
+        assert gdx['sym_11'].num_dims == 2
+        assert gdx['sym_11'].num_records == 4 
+        # ETH@20181007 - Tried to test for some values being c_bool(False) in sym_11, but
+        # c_bool(True) != c_bool(True), so that makes it hard to test such things.
+        # Also, c_bool(False) appears to be interpreted as True in GDX. Ick and yikes.
+        assert gdx['sym_12'].num_dims == 2
+        assert gdx['sym_12'].num_records == 0
+        assert gdx['sym_13'].dims == ['u','q','c']
+        assert gdx['sym_13'].dataframe[gdxpds.gdx.GamsValueType.Upper.name].values[0] == 10.0
+        assert gdx['sym_14'].num_records == 3
+        assert gdx['sym_15'].num_dims == 4
+        assert gdx['sym_16'].num_dims == 2
+        assert gdx['sym_16'].num_records == 4 
+        assert gdx['sym_17'].num_dims == 2
+        assert gdx['sym_17'].num_records == 0