From ebf188033a66a321e1be98ebe2591a27a1cb4f5a Mon Sep 17 00:00:00 2001 From: very-doge-wow <95224950+very-doge-wow@users.noreply.github.com> Date: Sun, 17 Mar 2024 18:41:39 +0100 Subject: [PATCH] Ability to render nested keys (#75) * feat: ability to add docs to lower level keys * feat: ability to add docs to lower level keys * feat: ability to add docs to lower level keys * test: add unit tests * fix: indent of examples * fix: remove unneeded import --------- Co-authored-by: very-doge-wow --- reader/chart_reader.py | 92 +++++++++++- reader/chart_reader_test.py | 218 ++++++++++++++++++++++++++++ test/test-chart/values.yaml | 2 +- test/values-nested-docs/values.yaml | 19 +++ 4 files changed, 324 insertions(+), 7 deletions(-) create mode 100644 test/values-nested-docs/values.yaml diff --git a/reader/chart_reader.py b/reader/chart_reader.py index 83db485..214966a 100644 --- a/reader/chart_reader.py +++ b/reader/chart_reader.py @@ -1,3 +1,5 @@ +import re + import yaml import logging import os @@ -68,6 +70,76 @@ def generate_chart_metadata(doc: dict, helm_chart_path: str) -> dict: return doc +def get_value_from_yaml(parsed_yaml: dict, full_path: str) -> dict: + """ + Takes a full yaml path such as first.second.third and gets + the associated yaml value from the dictionary. Preserves + toplevel keys while doing so. + Parameters: + parsed_yaml (dict): data structure from which to read + full_path (str): full path to desired values + Returns: + result (dict): Generated data structure. + """ + keys = full_path.split('.') + result = {} + current = result + for key in keys[:-1]: + current[key] = {} + current = current[key] + if key in parsed_yaml: + parsed_yaml = parsed_yaml[key] + else: + return result # Return if any intermediate key is not found + current[keys[-1]] = parsed_yaml.get(keys[-1]) # Get the value if exists + return result + + +def build_full_path(i: int, value_name_dirty: str, value_name_clean: str, values_lines: list) -> str: + """ + Takes a value name and its current index when traversing a values file + line by line and tries to determine the full path inside the yaml + document without parsing it. Will evaluate lines above the current + line and their respective indent until the toplevel is reached, all + the while building the full path. + Parameters: + i (int): current index from outer scope + value_name_dirty (str): the current value's name without having removed leading whitespace + value_name_clean (str): the current value's name sanitized + values_lines (list): list of all lines in the values document + Returns: + full_path (str): Full path to the currently evaluated value inside the document. + """ + # first element will always be the current key's name + full_path = value_name_clean + # check if whitespace before key is found + match = re.search(r'^(\s+).*$', value_name_dirty) + index = i + while match: + # count the indent + indent_num = match.group(0).count(' ') + # early exit if already on toplevel + if indent_num == 0: + return f"{upper_key}.{full_path}" + # iterate to the nearest key which is (closer to) top-level + while values_lines[index - 1].lstrip().startswith("#") or values_lines[index - 1].strip() == "": + # loop ignores empty lines and comments + index -= 1 + # loop terminates when next yaml key is found + index -= 1 + # index now points to the line with the key + value_name_dirty = values_lines[index].split(":")[0] + upper_key = value_name_dirty.strip() + # make sure the found key is actually closer to top-level than the first one by counting indent + match_new = re.search(r'^\s*', value_name_dirty) + if match_new: + indent_num_new = match.group(0).count(' ') + if indent_num_new < indent_num: + full_path = f"{upper_key}.{full_path}" + match = re.search(r'^\s*', value_name_dirty) + return full_path + + def generate_values_doc(doc: dict, helm_chart_path: str) -> dict: """ Reads stella doc strings from values.yaml and assigns them to a specific value entry. @@ -95,13 +167,16 @@ def generate_values_doc(doc: dict, helm_chart_path: str) -> dict: for index, line in enumerate(values_lines): if stella in line: # found a stella doc string + # get the indent + match = re.search(r'^(\s+).*$', line) + indent_num = match.group(0).count(' ')-1 if match else 0 doc_string = "" i = index # check if the next line still is a comment, if so add it to docstring - while values_lines[i + 1].startswith("#"): + while values_lines[i + 1].lstrip().startswith("#"): # remove first char (#) and add newline - calc = values_lines[i + 1].replace("#", "", 1) + "\n" - if calc[0] == " ": + calc = values_lines[i + 1].replace("#", "", 1).replace(" ", "", indent_num) + "\n" + if indent_num == 0 and calc[0] == " ": calc = calc.replace(" ", "", 1) doc_string += calc i += 1 @@ -109,8 +184,13 @@ def generate_values_doc(doc: dict, helm_chart_path: str) -> dict: while values_lines[i + 1].strip() == "": # if it is whitespace, ignore the line i += 1 + # when the loop is terminated, the nearest value name is extracted - value_name = values_lines[i + 1].split(":")[0].strip() + i += 1 + value_name_dirty = values_lines[i].split(":")[0] + value_name_sanitized = value_name_dirty.strip() + # if it is not a top-level value, we need to determine the entire yaml path to the element + full_path = build_full_path(i, value_name_dirty, value_name_sanitized, values_lines) # check if an example is present in the docstring example_delimiter = "-- example" @@ -125,9 +205,9 @@ def generate_values_doc(doc: dict, helm_chart_path: str) -> dict: # write the generated values to the output data structure doc["values"].append({ - "name": value_name, + "name": full_path, "description": doc_string, - "default": {value_name: values_yaml[value_name]}, + "default": get_value_from_yaml(values_yaml, full_path), "example": example.replace("|", "\\|") # escape pipe symbol to correctly render md table }) # also add doc entries for values that do not have stella docstrings diff --git a/reader/chart_reader_test.py b/reader/chart_reader_test.py index d01464c..c59db1a 100644 --- a/reader/chart_reader_test.py +++ b/reader/chart_reader_test.py @@ -1,3 +1,5 @@ +import yaml + import chart_reader from unittest.mock import Mock from hamcrest import assert_that, has_entries, contains_inanyorder @@ -191,6 +193,222 @@ def test_generate_values_comments_in_examples(): )) +def test_generate_values_docs_nested(): + doc = { + "name": "", + "appVersion": "", + "apiVersion": "", + "version": "", + "description": "", + "type": "", + "dependencies": [], + "values": [], + "templates": [], + "objects": [], + "commands": [], + } + result = chart_reader.generate_values_doc(doc, "test/values-nested-docs") + print(result) + assert_that(result["values"], contains_inanyorder( + {'name': 'image', 'description': 'which image to deploy\n', 'default': {'image': {'repository': 'nginx', 'pullPolicy': 'IfNotPresent', 'tag': ''}}, 'example': '\nimage:\n repository: very-doge-wow/stella\n pullPolicy: IfNotPresent\n'}, {'name': 'image.tag', 'description': 'Overrides the image tag whose default is the chart appVersion.\n', 'default': {'image': {'tag': ''}}, 'example': '\nimage:\n tag: "latest"\n'}, {'name': 'replicaCount', 'description': 'how many replicas to deploy\n', 'default': {'replicaCount': 1}, 'example': ''} + )) + + +def test_get_value_from_yaml(): + yaml_string = """first: + second: + third: "value" +""" + path = "first.second.third" + result = chart_reader.get_value_from_yaml(yaml.safe_load(yaml_string), path) + assert result == { + "first": { + "second": { + "third": "value" + } + } + } + + yaml_string = """first: 1 +""" + path = "first" + result = chart_reader.get_value_from_yaml(yaml.safe_load(yaml_string), path) + assert result == { + "first": 1 + } + + yaml_string = """first: + - name: lol + value: rofl +""" + path = "first" + result = chart_reader.get_value_from_yaml(yaml.safe_load(yaml_string), path) + assert result == { + "first": [{"name": "lol", "value": "rofl"}] + } + + yaml_string = """first: + second: + third: lol + fourth: rofl +""" + path = "first.fourth" + result = chart_reader.get_value_from_yaml(yaml.safe_load(yaml_string), path) + assert result == { + "first": { + "fourth": "rofl" + } + } + + yaml_string = """first: + second: + third: + fifth: uhuhu + fourth: rofl +""" + path = "first.second.fourth" + result = chart_reader.get_value_from_yaml(yaml.safe_load(yaml_string), path) + assert result == { + "first": { + "second": { + "fourth": "rofl" + } + } + } + + yaml_string = """first: + second: + third: + fifth: uhuhu + fourth: rofl +another: one +""" + path = "first.second.fourth" + result = chart_reader.get_value_from_yaml(yaml.safe_load(yaml_string), path) + assert result == { + "first": { + "second": { + "fourth": "rofl" + } + } + } + + yaml_string = """first: + second: + third: + fifth: uhuhu + fourth: rofl +another: one + """ + path = "another" + result = chart_reader.get_value_from_yaml(yaml.safe_load(yaml_string), path) + assert result == { + "another": "one" + } + + yaml_string = """first: {} + """ + path = "first" + result = chart_reader.get_value_from_yaml(yaml.safe_load(yaml_string), path) + assert result == { + "first": {} + } + + yaml_string = """first: [] + """ + path = "first" + result = chart_reader.get_value_from_yaml(yaml.safe_load(yaml_string), path) + assert result == { + "first": [] + } + + yaml_string = """first: "" + """ + path = "first" + result = chart_reader.get_value_from_yaml(yaml.safe_load(yaml_string), path) + assert result == { + "first": "" + } + + yaml_string = """first: '' + """ + path = "first" + result = chart_reader.get_value_from_yaml(yaml.safe_load(yaml_string), path) + assert result == { + "first": "" + } + + yaml_string = """first: + very: + nested: + indeed: + omg: + this: + is: + so: + nested: true + """ + path = "first" + result = chart_reader.get_value_from_yaml(yaml.safe_load(yaml_string), path) + assert result == { + "first": { + "very": { + "nested": { + "indeed": { + "omg": { + "this": { + "is": { + "so": { + "nested": True + } + } + } + } + } + } + } + } + } + + +def test_build_full_path(): + test_yaml = """--- +first: + element: "wow" + another: "one" + +emptydict: {} + +emptyarray: [] + +yet: + another: [] +""" + result = chart_reader.build_full_path(i=3, value_name_dirty=' another', value_name_clean='another', + values_lines=test_yaml.split('\n')) + assert result == "first.another" + + result = chart_reader.build_full_path(i=2, value_name_dirty=' element', value_name_clean='element', + values_lines=test_yaml.split('\n')) + assert result == "first.element" + + result = chart_reader.build_full_path(i=5, value_name_dirty='emptydict', value_name_clean='emptydict', + values_lines=test_yaml.split('\n')) + assert result == "emptydict" + + result = chart_reader.build_full_path(i=5, value_name_dirty='emptyarray', value_name_clean='emptyarray', + values_lines=test_yaml.split('\n')) + assert result == "emptyarray" + + result = chart_reader.build_full_path(i=9, value_name_dirty='yet', value_name_clean='yet', + values_lines=test_yaml.split('\n')) + assert result == "yet" + + result = chart_reader.build_full_path(i=10, value_name_dirty=' another', value_name_clean='another', + values_lines=test_yaml.split('\n')) + assert result == "yet.another" + + def test_generate_requirements(): doc = { "name": "", diff --git a/test/test-chart/values.yaml b/test/test-chart/values.yaml index fe64a93..c76040c 100644 --- a/test/test-chart/values.yaml +++ b/test/test-chart/values.yaml @@ -84,4 +84,4 @@ nodeSelector: {} tolerations: [] -affinity: {} +affinity: {} \ No newline at end of file diff --git a/test/values-nested-docs/values.yaml b/test/values-nested-docs/values.yaml new file mode 100644 index 0000000..ef49ee1 --- /dev/null +++ b/test/values-nested-docs/values.yaml @@ -0,0 +1,19 @@ +# -- stella +# how many replicas to deploy +replicaCount: 1 + +# -- stella +# which image to deploy +# -- example +# image: +# repository: very-doge-wow/stella +# pullPolicy: IfNotPresent +image: + repository: nginx + pullPolicy: IfNotPresent + # -- stella + # Overrides the image tag whose default is the chart appVersion. + # -- example + # image: + # tag: "latest" + tag: ""