From 35b5eb7429faee2af9ba32675ea115f2bca05f1c Mon Sep 17 00:00:00 2001 From: Mikkel Pedersen Date: Wed, 28 Aug 2024 23:25:33 +0200 Subject: [PATCH] fix(map): Add commands for env conditions and data collections --- ladybug_comfort/cli/__init__.py | 2 + ladybug_comfort/cli/datacollection.py | 74 ++++++++++++++++++ ladybug_comfort/cli/map.py | 107 ++++++++++++++++++++++++++ ladybug_comfort/map/_helper.py | 107 ++++++++++++++++++++++---- 4 files changed, 275 insertions(+), 15 deletions(-) create mode 100644 ladybug_comfort/cli/datacollection.py diff --git a/ladybug_comfort/cli/__init__.py b/ladybug_comfort/cli/__init__.py index 067b01bb..b8c316aa 100644 --- a/ladybug_comfort/cli/__init__.py +++ b/ladybug_comfort/cli/__init__.py @@ -6,6 +6,7 @@ from .sql import sql from .map import map from .mtx import mtx +from .datacollection import datacollection # command group for all comfort extension commands. @@ -20,6 +21,7 @@ def comfort(): comfort.add_command(sql) comfort.add_command(map) comfort.add_command(mtx) +comfort.add_command(datacollection, name='data-collection') # add comfort sub-group to ladybug CLI diff --git a/ladybug_comfort/cli/datacollection.py b/ladybug_comfort/cli/datacollection.py new file mode 100644 index 00000000..4d8ad9bd --- /dev/null +++ b/ladybug_comfort/cli/datacollection.py @@ -0,0 +1,74 @@ +"""Commands to work with data collections.""" +import sys +import logging +import numpy as np +from pathlib import Path +import click +import json + +from ladybug.datacollection import HourlyContinuousCollection, HourlyDiscontinuousCollection +from ladybug.header import Header +from ladybug.datautil import collections_to_csv + + +_logger = logging.getLogger(__name__) + + +@click.group(help='Commands to work with data collections.') +def datacollection(): + pass + + +@datacollection.command('folder-to-datacollections') +@click.argument( + 'folder', type=click.Path(exists=True, dir_okay=True, resolve_path=True) +) +@click.argument( + 'data-type', type=click.Path(exists=True, dir_okay=False, resolve_path=True) +) +@click.option( + '--sub-folder', '-sf', type=click.STRING, default='datacollections', + show_default=True +) +def folder_to_datacollections(folder, data_type, sub_folder): + """Read an npy file and convert every row to a data collection. + + The data collections will be saved as CSV files in subfolder. + + \b + Args: + npy-file: Path to npy file. + data-type: A JSON file with the data type. + grid-name: The name of the grid. This is used in the metadata of the + header. + """ + with open(Path(folder, 'grids_info.json')) as json_file: + grid_list = json.load(json_file) + with open(data_type) as json_file: + data_header = Header.from_dict(json.load(json_file)) + a_per = data_header.analysis_period + continuous = True if a_per.st_hour == 0 and a_per.end_hour == 23 else False + if not continuous: + dates = a_per.datetimes + try: + for grid in grid_list: + grid_name = grid['full_id'] if 'full_id' in grid else 'id' + metadata = {'grid': grid_name} + grid_file = Path(folder, '{}.npy'.format(grid_name)) + data_matrix = np.load(grid_file).tolist() + grid_data = [] + for i, row in enumerate(data_matrix): + header = data_header.duplicate() + header.metadata = metadata.copy() + header.metadata['sensor_index'] = i + data = HourlyContinuousCollection(header, row) if continuous else \ + HourlyDiscontinuousCollection(header, row, dates) + grid_data.append(data) + + file_name = grid_name + '.csv' + collections_to_csv(grid_data, Path(folder, sub_folder), file_name) + except Exception: + _logger.exception('Failed to convert folder of files to data collections.') + sys.exit(1) + else: + sys.exit(0) diff --git a/ladybug_comfort/cli/map.py b/ladybug_comfort/cli/map.py index b70bfe78..4b5fb294 100644 --- a/ladybug_comfort/cli/map.py +++ b/ladybug_comfort/cli/map.py @@ -22,12 +22,17 @@ ThermalCondition, ThermalConditionElevenPoint from ladybug.datatype.temperaturedelta import OperativeTemperatureDelta from ladybug.datatype.fraction import Fraction +from ladybug.datatype.temperature import AirTemperature, \ + MeanRadiantTemperature, RadiantTemperature +from ladybug.datatype.temperaturedelta import RadiantTemperatureDelta +from ladybug.datatype.fraction import RelativeHumidity from ladybug_comfort.map.irr import irradiance_contrib_map from ladybug_comfort.map.mrt import shortwave_mrt_map, longwave_mrt_map from ladybug_comfort.map.air import air_map from ladybug_comfort.map.tcp import tcp_model_schedules, tcp_total from ladybug_comfort.map._enclosure import _parse_enclosure_info, _values_to_data +from ladybug_comfort.map._helper import restore_original_distribution from ladybug_comfort.collection.pmv import PMV, _PMVnoSET from ladybug_comfort.collection.adaptive import Adaptive, PrevailingTemperature from ladybug_comfort.collection.utci import UTCI @@ -36,6 +41,7 @@ load_pmv_par_str, load_adaptive_par_str, load_utci_par_str, \ load_solarcal_par_str, thermal_map_csv, _data_to_ill, set_smallest_dtype + _logger = logging.getLogger(__name__) @@ -920,6 +926,107 @@ def tcp(condition_csv, enclosure_info, schedule, occ_schedule_json, folder, log_ sys.exit(0) +@map.command('restructure-env-conditions') +@click.argument( + 'folder', type=click.Path(exists=True, dir_okay=True, resolve_path=True) +) +@click.argument( + 'dest-folder', type=click.Path(exists=False, dir_okay=True, resolve_path=True) +) +@click.argument( + 'sub-path', type=click.STRING +) +def restructure_env_conditions(folder, dest_folder, sub_path): + """Read an npy file and convert every row to a data collection. + + This command reads a NumPy array from a npy file and sends it to stdout. + + \b + Args: + folder: Folder with environmental conditions (initial results). + dest_folder: Destination folder for writing the results. + sub_path: Sub path for the metric (mrt, air_temperature, longwave_mrt, + shortwave_mrt, rel_humidity)- + """ + try: + source_folder = os.path.join(folder, sub_path) + dest_folder = os.path.join(folder, 'final', sub_path) + if sub_path == 'mrt': + source_folders = [os.path.join(folder, 'longwave_mrt'), + os.path.join(folder, 'shortwave_mrt')] + dest_folders = [os.path.join(folder, 'final', 'longwave_mrt'), + os.path.join(folder, 'final', 'shortwave_mrt')] + else: + assert os.path.isdir(source_folder), \ + 'Metric "{}" does not exist for this comfort study.'.format(sub_path) + source_folders, dest_folders = [source_folder], [dest_folder] + + # restructure the results to align with the sensor grids + dist_info = os.path.join(folder, '_redist_info.json') + for src_f, dst_f in zip(source_folders, dest_folders): + if not os.path.isdir(dst_f): + os.makedirs(dst_f) + restore_original_distribution( + src_f, dst_f, extension='csv', dist_info=dist_info, + output_extension='csv', as_text=True, fmt='%.12f', + delimiter='comma') + grid_info_src = os.path.join(folder, 'grids_info.json') + grid_info_dst = os.path.join(dst_f, 'grids_info.json') + shutil.copyfile(grid_info_src, grid_info_dst) + data_header = create_result_header(folder, os.path.split(dst_f)[-1]) + result_info_path = os.path.join(dst_f, 'results_info.json') + with open(result_info_path, 'w') as fp: + json.dump(data_header.to_dict(), fp, indent=4) + # if MRT was requested, sum together the longwave and shortwave + if sub_path == 'mrt': + sum_matrices(dest_folders[0], dest_folders[1], dest_folder) + data_header = create_result_header(folder, sub_path) + result_info_path = os.path.join(dest_folder, 'results_info.json') + with open(result_info_path, 'w') as fp: + json.dump(data_header.to_dict(), fp, indent=4) + except Exception: + _logger.exception('Failed to restructure environmental conditions.') + sys.exit(1) + else: + sys.exit(0) + + +def create_result_header(env_conds, sub_path): + """Create a DataCollection Header for a given metric.""" + with open(os.path.join(env_conds, 'results_info.json')) as json_file: + base_head = Header.from_dict(json.load(json_file)) + if sub_path == 'mrt': + return Header(MeanRadiantTemperature(), 'C', base_head.analysis_period) + elif sub_path == 'air_temperature': + return Header(AirTemperature(), 'C', base_head.analysis_period) + elif sub_path == 'longwave_mrt': + return Header(RadiantTemperature(), 'C', base_head.analysis_period) + elif sub_path == 'shortwave_mrt': + return Header(RadiantTemperatureDelta(), 'dC', base_head.analysis_period) + elif sub_path == 'rel_humidity': + return Header(RelativeHumidity(), '%', base_head.analysis_period) + + +def sum_matrices(mtxs_1, mtxs_2, dest_dir): + """Sum together matrices of two folders.""" + if not os.path.isdir(dest_dir): + os.makedirs(dest_dir) + for mtx_file in os.listdir(mtxs_1): + if mtx_file.endswith('.csv'): + mtx_file1 = os.path.join(mtxs_1, mtx_file) + mtx_file2 = os.path.join(mtxs_2, mtx_file) + matrix_1 = np.loadtxt(mtx_file1, dtype=np.float32, delimiter=',') + matrix_2 = np.loadtxt(mtx_file2, dtype=np.float32, delimiter=',') + data = matrix_1 + matrix_2 + csv_path = os.path.join(dest_dir, mtx_file) + np.savetxt(csv_path, data, fmt='%.12f', delimiter=',') + elif mtx_file == 'grids_info.json': + shutil.copyfile( + os.path.join(mtxs_1, mtx_file), + os.path.join(dest_dir, mtx_file) + ) + + def _tcp_config(): """Return vtk-config for a thermal comfort map.""" return { diff --git a/ladybug_comfort/map/_helper.py b/ladybug_comfort/map/_helper.py index 1e1a2ae9..557a51b1 100644 --- a/ladybug_comfort/map/_helper.py +++ b/ladybug_comfort/map/_helper.py @@ -1,4 +1,6 @@ """A collection of helper functions for the map sub-package.""" +import json +from pathlib import Path import numpy as np @@ -71,21 +73,6 @@ def binary_to_array( Returns: A NumPy array. """ - with open(binary_file, 'rb') as file: - # check if file is NumPy file - numpy_header = file.read(6) - if numpy_header.startswith(b'\x93NUMPY'): - file.seek(0) - array = np.load(file) - return array - file.seek(0) - # check if file has Radiance header, if not it is a text file - radiance_header = file.read(10).decode('utf-8') - if radiance_header != '#?RADIANCE': - file.seek(0) - array = np.genfromtxt(file, dtype=np.float32) - return array - if (nrows or ncols or ncomp or fmt) is None: # get nrows, ncols and header line count nrows, ncols, ncomp, line_count, fmt = binary_mtx_dimension(binary_file) @@ -120,3 +107,93 @@ def load_matrix(matrix_file, delimiter=','): array = np.load(matrix_file) return array + + +def restore_original_distribution( + input_folder, output_folder, extension='npy', dist_info=None, + output_extension='ill', as_text=False, fmt='%.2f', input_delimiter=',', + delimiter='tab'): + """Restructure files to the original distribution based on the distribution info. + + It will assume that the files in the input folder are NumPy files. However, + if it fails to load the files as arrays it will try to load from binary + Radiance files to array. + + Args: + input_folder: Path to input folder. + output_folder: Path to the new restructured folder + extension: Extension of the files to collect data from. Default is ``npy`` for + NumPy files. Another common extension is ``ill`` for the results of daylight + studies. + dist_info: Path to dist_info.json file. If None, the function will try to load + ``_redist_info.json`` file from inside the input_folder. (Default: None). + output_extension: Output file extension. This is only used if as_text + is set to True. Otherwise the output extension will be ```npy``. + as_text: Set to True if the output files should be saved as text instead + of NumPy files. + fmt: Format for the output files when saved as text. + input_delimiter: Delimiter for the input files. This is used only if the + input files are text files. + delimiter: Delimiter for the output files when saved as text. + """ + if not dist_info: + _redist_info_file = Path(input_folder, '_redist_info.json') + else: + _redist_info_file = Path(dist_info) + + assert _redist_info_file.is_file(), 'Failed to find %s' % _redist_info_file + + with open(_redist_info_file) as inf: + data = json.load(inf) + + # create output folder + output_folder = Path(output_folder) + if not output_folder.is_dir(): + output_folder.mkdir(parents=True, exist_ok=True) + + src_file = Path() + for f in data: + output_file = Path(output_folder, f['identifier']) + # ensure the new folder is created. in case the identifier has a subfolder + parent_folder = output_file.parent + if not parent_folder.is_dir(): + parent_folder.mkdir() + + out_arrays = [] + for src_info in f['dist_info']: + st = src_info['st_ln'] + end = src_info['end_ln'] + new_file = Path(input_folder, '%s.%s' % (src_info['identifier'], extension)) + if not new_file.samefile(src_file): + src_file = new_file + try: + array = np.load(src_file) + except: + try: + array = binary_to_array(src_file) + except: + try: + array = np.loadtxt( + src_file, delimiter=input_delimiter) + except Exception: + raise RuntimeError( + f'Failed to load input file "{src_file}"') + slice_array = array[st:end+1,:] + + out_arrays.append(slice_array) + + out_array = np.concatenate(out_arrays) + # save numpy array, .npy extension is added automatically + if not as_text: + np.save(output_file, out_array) + else: + if output_extension.startswith('.'): + output_extension = output_extension[1:] + if delimiter == 'tab': + delimiter = '\t' + elif delimiter == 'space': + delimiter = ' ' + elif delimiter == 'comma': + delimiter = ',' + np.savetxt(output_file.with_suffix(f'.{output_extension}'), + out_array, fmt=fmt, delimiter=delimiter)