diff --git a/dbt_docs_to_notion.py b/dbt_docs_to_notion.py index 832e7a0..5abc85f 100644 --- a/dbt_docs_to_notion.py +++ b/dbt_docs_to_notion.py @@ -31,46 +31,31 @@ def make_request(endpoint, querystring='', method='GET', **request_kwargs): return resp.json() -def get_path_or_empty(parent_object, path_array, zero_value=''): +def get_paths_or_empty(parent_object, paths_array, zero_value=''): + """Used for catalog_nodes accesses, since structure is variable""" + for path in paths_array: obj = parent_object - for el in path_array: - if el not in obj: - return zero_value - obj = obj[el] + for el in path: + if el not in obj: + obj = zero_value + break + obj = obj[el] + if obj != zero_value: + return obj - return obj - - -def get_num_rows(catalog_nodes, model_name): - zero_value = NUMERIC_ZERO_VALUE - keys = ['num_rows', 'row_count'] - for key in keys: - num_rows = get_path_or_empty(catalog_nodes, [model_name, 'stats', key, 'value'], NUMERIC_ZERO_VALUE) - if num_rows != NUMERIC_ZERO_VALUE: - return num_rows - - return NUMERIC_ZERO_VALUE - - -def get_bytes(catalog_nodes, model_name): - zero_value = NUMERIC_ZERO_VALUE - keys = ['num_bytes', 'bytes'] - for key in keys: - num_rows = get_path_or_empty(catalog_nodes, [model_name, 'stats', key, 'value'], NUMERIC_ZERO_VALUE) - if num_rows != NUMERIC_ZERO_VALUE: - return num_rows - - return NUMERIC_ZERO_VALUE + return zero_value def get_owner(data, catalog_nodes, model_name): - # Check for an owner field explicitly named in the DBT Config - # If none present, fall back to database table owner - owner = get_path_or_empty(data, ['config', 'meta', 'owner'], None) - if owner != None: - return owner + """ + Check for an owner field explicitly named in the DBT Config + If none present, fall back to database table owner + """ + owner = get_paths_or_empty(data, [['config', 'meta', 'owner']], None) + if owner is not None: + return owner - return get_path_or_empty(catalog_nodes, [model_name, 'metadata', 'owner'], '') + return get_paths_or_empty(catalog_nodes, [[model_name, 'metadata', 'owner']], '') def main(): @@ -153,7 +138,7 @@ def main(): } } - print('created creating database') + print('creating database') database_creation_resp = make_request( endpoint='databases/', querystring='', @@ -207,7 +192,11 @@ def main(): } } ] - col_names_and_data = list(get_path_or_empty(catalog_nodes, [model_name, 'columns'], {}).items()) + col_names_and_data = list(get_paths_or_empty( + catalog_nodes, + [[model_name, 'columns']], + {} + ).items()) for (col_name, col_data) in col_names_and_data[:98]: # notion api limit is 100 table rows columns_table_children_obj.append( { @@ -427,10 +416,20 @@ def main(): ] }, "Approx Rows": { - "number": get_num_rows(catalog_nodes, model_name) + "number": get_paths_or_empty( + catalog_nodes, + [[model_name, 'stats', 'num_rows', 'value'], + [model_name, 'stats', 'row_count', 'value']], + NUMERIC_ZERO_VALUE + ) }, "Approx GB": { - "number":get_bytes(catalog_nodes, model_name) /1e9 + "number": get_paths_or_empty( + catalog_nodes, + [[model_name, 'stats', 'bytes', 'value'], + [model_name, 'stats', 'num_bytes', 'value']], + NUMERIC_ZERO_VALUE + ) / 1e9 }, "Depends On": { "rich_text": [