Skip to content

Commit

Permalink
Modify stats calculated. (#24)
Browse files Browse the repository at this point in the history 
Signed-off-by: Caroline Russell <caroline@appthreat.dev>
  • Loading branch information
@cerrussell
cerrussell authored Jun 11, 2024
1 parent fae33b0 commit 426a7fc
Show file tree
Hide file tree
Showing 4 changed files with 134 additions and 72 deletions.
107 changes: 80 additions & 27 deletions custom_json_diff/bom_diff_template.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,6 @@
<head>
<title>BOM Diff Summary</title>
<style>
th, td {
border: 1px solid black;
padding: 15px;
vertical-align: top;
text-align: left;
}
.table_summary {
display: grid;
Expand All @@ -17,7 +11,19 @@
.table_summary2 {
display: grid;
grid-template-columns: 75% 20% 5%;
grid-template-columns: 72% 23% 5%;
}
.table_summary3 {
display: grid;
grid-template-columns: 30% 40% 30%;
}
th, td {
border: 1px solid black;
padding: 5px;
vertical-align: top;
text-align: left;
}
</style>
Expand All @@ -28,19 +34,21 @@
<div>
<table style="width: 100%;">
<tr>
<th colspan="2" style="text-align: center; background-color: #d7ddde">Overall
<th colspan="2"
style="text-align: center; background-color: #d7ddde; border: 1px solid black;">
Overall
Summary
</th>
</tr>
{% for item in stats %}
{% for item in stats["common_summary"] %}
<tr>
<td>{{ item[0] }}</td>
<td style="text-align: right">{{ item[1] }}</td>

</tr>
{% endfor %}
<tr>
<td>Metadata matched</td>
<td>Metadata matched:</td>
{% if metadata %}
<td style="text-align: right">✖</td>
{% endif %}
Expand All @@ -50,16 +58,17 @@

</tr>
</table>

</div>
</div>
<div><h1 style="text-align:center;">BOM Diff Summary</h1></div>
<div class="table_summary">
<div></div>
<div>
<table class="master_table" style="width: 100%">
<table style="width: 100%">
<tr>
<th colspan="3" style="text-align: center; background-color: #d7ddde">Component
<th colspan="3"
style="text-align: center; background-color: #d7ddde; border: 1px solid black;">
Component
Diff
</th>
</tr>
Expand Down Expand Up @@ -228,29 +237,36 @@
<summary>{{ item['short_ref'] }}</summary>
<ul>
<li>ref: {{ item['ref'] }}</li>
<li>dependencies:
{% if item['dependsOn']|length > 0 %}
<li>dependencies:</li>
<ul>
{% for dep in item['dependsOn'] %}
<li>{{ dep }}</li>
{% endfor %}
</ul>
</li>
{% endif %}
{% if item['dependsOn']|length == 0 %}
<li>dependencies: N/A</li>
{% endif %}
</ul>

</details>
{% endfor %}</td>
<td>{% for item in diff_deps_2 %}
<details>
<summary>{{ item['short_ref'] }}</summary>
<ul>
<li>ref: {{ item['ref'] }}</li>
<li>dependencies:
{% if item['dependsOn']|length > 0 %}
<li>dependencies:</li>
<ul>
{% for dep in item['dependsOn'] %}
<li>{{ dep }}</li>
{% endfor %}
</ul>
</li>
{% endif %}
{% if item['dependsOn']|length == 0 %}
<li>dependencies: N/A</li>
{% endif %}
</ul>

</details>
Expand All @@ -266,9 +282,11 @@
<div class="table_summary">
<div></div>
<div>
<table class="master_table" style="width: 100%">
<table style="width: 100%">
<tr>
<th colspan="2" style="text-align: center; background-color: #d7ddde">Common
<th colspan="2"
style="text-align: center; background-color: #d7ddde; border: 1px solid black;">
Common
Components
</th>
</tr>
Expand Down Expand Up @@ -372,13 +390,18 @@
<summary>{{ item['short_ref'] }}</summary>
<ul>
<li>ref: {{ item['ref'] }}</li>
<li>dependencies:
<ul>
{% for dep in item['dependsOn'] %}
<li>{{ dep }}</li>
{% endfor %}
</ul>
</li>
{% if item['dependsOn']|length >0 %}
<li>dependencies:
<ul>
{% for dep in item['dependsOn'] %}
<li>{{ dep }}</li>
{% endfor %}
</ul>
</li>
{% endif %}
{% if item['dependsOn']|length == 0 %}
<li>dependencies: N/A</li>
{% endif %}
</ul>

</details>
Expand All @@ -391,5 +414,35 @@
<div></div>
</div>
<p></p>
<div class="table_summary3">
<div></div>
<div>
<table style="width: 100%">
<tr>
<th colspan="3"
style="text-align: center; background-color: #d7ddde; border: 1px solid black;">
Matched Components Breakdown
</th>
</tr>
<tr>
<td style="width: 20%"></td>
<th style="text-align: center; width: 40%">{{ bom_1 }}</th>
<th style="text-align: center; width: 40%">{{ bom_2 }}</th>
</tr>
{% for key, value in stats["breakdown"]|items %}
<tr>
<th>{{ key }}</th>
<td style="text-align: center; vertical-align: middle">{{ value[0] }}</td>
<td style="text-align: center; vertical-align: middle">{{ value[1] }}</td>
</tr>
{% endfor %}
<tr>

</tr>
</table>
</div>
<div></div>
</div>
<p></p>
</body>
</html>
75 changes: 36 additions & 39 deletions custom_json_diff/custom_diff.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,22 +15,30 @@
logger = logging.getLogger(__name__)


def calculate_pcts(diff_stats: Dict, j1: BomDicts, j2: BomDicts) -> List[list[str]]:
def calculate_pcts(diffs: Dict, j1: BomDicts, j2: BomDicts) -> Dict:
j1_counts = j1.generate_counts()
j2_counts = j2.generate_counts()
result = [
[f"Common {key} matched: ", f"{value}"]
for key, value in diff_stats["common"].items()
]
result.extend([
[f"BOM 1 {key} not matched: ", f"{j1_counts[key] - value}/{j1_counts[key]}"]
for key, value in diff_stats["common"].items()
])
result.extend([
[f"BOM 2 {key} not matched: ", f"{j2_counts[key] - value}/{j2_counts[key]}"]
for key, value in diff_stats["common"].items()
])
return [i for i in result if not i[1].startswith("0")]
common_counts = generate_counts(diffs["common_summary"])
result = []
for key, value in common_counts.items():
total = j1_counts[key] + j2_counts[key]
if not total == 0:
result.append([f"Common {key} matched: ", f"{value} ({round(((value*2)/total) * 100, 2)})%"])

result_2 = summarize_diffs({}, generate_counts(diffs["diff_summary"][j1.filename]), j1_counts)
result_2 = summarize_diffs(result_2, generate_counts(diffs["diff_summary"][j2.filename]), j2_counts)
return {"common_summary": result, "breakdown": result_2}


def summarize_diffs(result: Dict, diff_counts: Dict, bom_counts: Dict) -> Dict:
for key, value in diff_counts.items():
found = bom_counts[key] - value
if not bom_counts[key] == 0:
if result.get(key):
result[key].append(f"{found}/{bom_counts[key]} ({round(((found) / bom_counts[key]) * 100, 2)}%)")
else:
result[key] = [f"{found}/{bom_counts[key]} ({round(((found) / bom_counts[key]) * 100, 2)}%)"]
return result


def check_regex(regex_keys: Set[re.Pattern], key: str) -> bool:
Expand All @@ -57,14 +65,14 @@ def export_html_report(outfile: str, diffs: Dict, j1: BomDicts, j2: BomDicts, op
template = tmpl_file.read()
jinja_env = Environment(autoescape=False)
jinja_tmpl = jinja_env.from_string(template)
purl_regex = re.compile(r"[^/]+@[^?\s]+")
purl_regex = re.compile(r"[^/]+/[^/]+@[^?\s]+")
diffs["diff_summary"][options.file_1]["dependencies"] = parse_purls(
diffs["diff_summary"][options.file_1].get("dependencies", []), purl_regex)
diffs["diff_summary"][options.file_2]["dependencies"] = parse_purls(
diffs["diff_summary"][options.file_2].get("dependencies", []), purl_regex)
diffs["common_summary"]["dependencies"] = parse_purls(
diffs["common_summary"].get("dependencies", []), purl_regex)
stats_summary = calculate_pcts(generate_diff_counts(diffs, j1.options.file_2), j1, j2)
stats_summary = calculate_pcts(diffs, j1, j2)
metadata_results = bool(
diffs["diff_summary"][options.file_1].get("misc_data", {}) or
diffs["diff_summary"][options.file_2].get("misc_data", {})
Expand All @@ -75,15 +83,15 @@ def export_html_report(outfile: str, diffs: Dict, j1: BomDicts, j2: BomDicts, op
common_services=diffs["common_summary"].get("services", []),
common_deps=diffs["common_summary"].get("dependencies", []),
common_apps=diffs["common_summary"].get("components", {}).get("applications", []),
common_other=diffs["common_summary"].get("components", {}).get("other_types", []),
common_other=diffs["common_summary"].get("components", {}).get("other_components", []),
diff_lib_1=diffs["diff_summary"].get(options.file_1, {}).get("components", {}).get("libraries", []),
diff_lib_2=diffs["diff_summary"].get(options.file_2, {}).get("components", {}).get("libraries", []),
diff_frameworks_1=diffs["diff_summary"].get(options.file_1, {}).get("components", {}).get("frameworks", []),
diff_frameworks_2=diffs["diff_summary"].get(options.file_2, {}).get("components", {}).get("frameworks", []),
diff_apps_1=diffs["diff_summary"].get(options.file_1, {}).get("components", {}).get("applications", []),
diff_apps_2=diffs["diff_summary"].get(options.file_2, {}).get("components", {}).get("applications", []),
diff_other_1=diffs["diff_summary"].get(options.file_1, {}).get("components", {}).get("other_types", []),
diff_other_2=diffs["diff_summary"].get(options.file_2, {}).get("components", {}).get("other_types", []),
diff_other_1=diffs["diff_summary"].get(options.file_1, {}).get("components", {}).get("other_components", []),
diff_other_2=diffs["diff_summary"].get(options.file_2, {}).get("components", {}).get("other_components", []),
diff_services_1=diffs["diff_summary"].get(options.file_1, {}).get("services", []),
diff_services_2=diffs["diff_summary"].get(options.file_2, {}).get("services", []),
diff_deps_1=diffs["diff_summary"].get(options.file_1, {}).get("dependencies", []),
Expand All @@ -110,24 +118,15 @@ def filter_dict(data: Dict, options: Options) -> FlatDicts:
return FlatDicts(data).filter_out_keys(options.exclude)


def generate_diff_counts(diffs, f2: str) -> Dict:
return {"common": {"components": len(
diffs["common_summary"].get("components", {}).get("libraries", [])) + len(
diffs["common_summary"].get("components", {}).get("frameworks", [])) + len(
diffs["common_summary"].get("components", {}).get("applications", [])) + len(
diffs["common_summary"].get("components", {}).get("other_types", [])),
"services": len(diffs["common_summary"].get("services", [])),
"dependencies": len(diffs["common_summary"].get("dependencies", []))},
"diff": {"components": len(
diffs["diff_summary"].get(f2, {}).get("components", {}).get("libraries",
[])) + len(
diffs["diff_summary"].get(f2, {}).get("components", {}).get("frameworks",
[])) + len(
diffs["diff_summary"].get(f2, {}).get("components", {}).get("applications",
[])) + len(
diffs["diff_summary"].get(f2, {}).get("components", {}).get("other_types", [])), },
"services": len(diffs["diff_summary"].get(f2, {}).get("services", [])),
"dependencies": len(diffs["diff_summary"].get(f2, {}).get("dependencies", []))}
def generate_counts(data: Dict) -> Dict:
result = {"libraries": len(data.get("components", {}).get("libraries", [])),
"frameworks": len(data.get("components", {}).get("frameworks", [])),
"applications": len(data.get("components", {}).get("applications", [])),
"other_components": len(data.get("components", {}).get("other_components", []))}
result["components"] = sum(result.values())
result |= {"services": len(data.get("services", [])),
"dependencies": len(data.get("dependencies", []))}
return result


def get_diff(j1: FlatDicts, j2: FlatDicts, options: Options) -> Dict:
Expand Down Expand Up @@ -164,8 +163,6 @@ def load_json(json_file: str, options: Options) -> FlatDicts | BomDicts:


def parse_purls(deps: List[Dict], regex: re.Pattern) -> List[Dict]:
if not deps:
return deps
for i in deps:
i["short_ref"] = match[0] if (match := regex.findall(i["ref"])) else i["ref"]
return deps
Expand Down
22 changes: 17 additions & 5 deletions custom_json_diff/custom_diff_classes.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,10 +141,22 @@ def intersection(self, other, title: str = "") -> "BomDicts":
return new_bom_dict

def generate_counts(self) -> Dict:
return {
"filename": self.filename, "components": len(self.components),
"services": len(self.services), "dependencies": len(self.dependencies)
}
lib = 0
frameworks = 0
apps = 0
other = 0
for i in self.components:
if i.component_type == "library":
lib += 1
elif i.component_type == "framework":
frameworks += 1
elif i.component_type == "application":
apps += 1
else:
other += 1
return {"components": len(self.components), "applications": apps,
"frameworks": frameworks, "libraries": lib, "other_components": other,
"services": len(self.services), "dependencies": len(self.dependencies)}

def to_summary(self) -> Dict:
summary: Dict = {self.filename: {}}
Expand All @@ -157,7 +169,7 @@ def to_summary(self) -> Dict:
i.original_data for i in self.components if i.component_type == "framework"],
"applications": [i.original_data for i in self.components if
i.component_type == "application"],
"other_types": [i.original_data for i in self.components if
"other_components": [i.original_data for i in self.components if
i.component_type not in ("library", "framework", "application")],
}
}
Expand Down
2 changes: 1 addition & 1 deletion test/test_data.json
View file

Large diffs are not rendered by default.

0 comments on commit 426a7fc

Please sign in to comment.