Merge pull request #13 from ezyang/jjwu_compilation_metrics

Add compilation metrics parsing
ezyang · Apr 4, 2024 · f091d1d · f091d1d
2 parents d587e70 + 5a7cd0c
commit f091d1d
Show file tree

Hide file tree

Showing 8 changed files with 379 additions and 0 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -18,6 +18,7 @@ path = "src/cli.rs"
 [dependencies]
 anyhow = "1.0.75"
 base16ct = "0.2.0"
+chrono = "0.4"
 clap = { version = "4.5.2", features = ["derive"] }
 fxhash = "0.2.1"
 html-escape = "0.2.5"

diff --git a/src/lib.rs b/src/lib.rs
@@ -71,6 +71,8 @@ pub fn parse_path(path: &PathBuf, config: ParseConfig) -> anyhow::Result<ParseOu
     tt.add_formatter("format_unescaped", tinytemplate::format_unescaped);
     tt.add_template("index.html", TEMPLATE_INDEX)?;
     tt.add_template("dynamo_guards.html", TEMPLATE_DYNAMO_GUARDS)?;
+    tt.add_template("compilation_metrics.html", TEMPLATE_COMPILATION_METRICS)?;
+
 
     // NB: Sometimes, the log output we get from Logarithm stutters with a blank line.
     // Filter them out, they're never valid (a blank line in payload will still be \t)

diff --git a/src/parsers.rs b/src/parsers.rs
@@ -188,6 +188,34 @@ impl StructuredLogParser for OptimizeDdpSplitChildParser {
     }
 }
 
+pub struct CompilationMetricsParser<'t> {
+    tt: &'t TinyTemplate<'t>,
+}
+impl StructuredLogParser for CompilationMetricsParser<'_> {
+    fn name(&self) -> &'static str {
+        "compilation_metrics"
+    }
+    fn get_metadata<'e>(&self, e: &'e Envelope) -> Option<Metadata<'e>> {
+        e.compilation_metrics.as_ref().map(|m| Metadata::CompilationMetrics(m))
+    }
+    fn parse<'e>(&self,
+            lineno: usize,
+            metrics: Metadata<'e>,
+            _rank: Option<u32>,
+            compile_id: &Option<CompileId>,
+            _payload: &str
+    ) -> anyhow::Result<ParseOutput> {
+        let filename = format!("{}.html", self.name());
+        if let Metadata::CompilationMetrics(m) = metrics {
+            let output = self.tt.render(&filename, &m)?;
+            simple_file_output(&filename, lineno, compile_id, &output)
+        } else {
+            Err(anyhow::anyhow!("Expected CompilationMetrics metadata"))
+        }
+
+    }
+}
+
 // Register your parser here
 pub fn default_parsers<'t>(tt: &'t TinyTemplate<'t>) -> Vec<Box<dyn StructuredLogParser + 't>> {
     // We need to use Box wrappers here because vecs in Rust need to have known size
@@ -202,6 +230,7 @@ pub fn default_parsers<'t>(tt: &'t TinyTemplate<'t>) -> Vec<Box<dyn StructuredLo
         Box::new(DynamoGuardParser { tt }),
         Box::new(InductorOutputCodeParser),
         Box::new(OptimizeDdpSplitChildParser),
+        Box::new(CompilationMetricsParser { tt }), // TODO: use own tt instances
     ];
 
     result

diff --git a/src/templates.rs b/src/templates.rs
@@ -89,3 +89,62 @@ Build products below:
 </body>
 </html>
 "#;
+
+pub static TEMPLATE_COMPILATION_METRICS : &str = r#"
+<html>
+<head>
+    <title>Compilation Metrics</title>
+</head>
+<body>
+    <h1>Compilation Info</h1>
+    <h2>Dynamo Output:</h2>
+    <object data="dynamo_output_graph.txt" style="width:80%; height:auto">
+    <a href="dynamo_output_graph.txt"> dynamo_output_graph.txt </a>
+    </object>
+    <h2>Compile Time(seconds)</h2>
+    <p>Entire Frame <abbr title="Total time spent in convert_frame function">[?]</abbr>: {entire_frame_compile_time_s}</div>
+    <p>Backend <abbr title="Time spent running the backend compiler">[?]</abbr>: {backend_compile_time_s}</div>
+    {{ if inductor_compile_time_s }}
+    <p>Inductor <abbr title="Total time spent running inductor">[?]</abbr>: {inductor_compile_time_s}</div>
+    {{ endif }}
+    {{ if code_gen_time_s }}
+    <p>Code Gen Time: {code_gen_time_s}</p>
+    {{ endif}}
+    <div>Dynamo Time Before Restart <abbr title="Total time spent restarting dynamo analysis">[?]</abbr>: {dynamo_time_before_restart_s}</div>
+    <h2>Restarts and Failures</h2>
+    {{ if fail_type }}
+    <p>Failure Exception: {fail_type}</p>
+    <p>Failure Reason: {fail_reason}</p>
+    <p>In file {fail_user_frame_filename}, line {fail_user_frame_lineno}</p>
+    {{ else }}
+    <p> No failures! </p>
+    {{ endif }}
+    {{ if restart_reasons }}
+    <p>Restart Reasons:<p>
+    {{ for restart_reason in restart_reasons }}
+     <li> <code> {restart_reason } </code> </li>
+    {{ endfor }}
+    {{ else }}
+    <p> No restarts! </p>
+    {{ endif }}
+    <h2>Cache Metrics</h2>
+    <p>Cache Size: {cache_size}</p>
+    <p>Accumulated Cache Size: {accumulated_cache_size}</p>
+    <h2>Graph Metrics</h2>
+    <p><a href='dynamo_guards.html'>Guard</a> Count: {guard_count}</p>
+    <p>Shape Env Guards: {shape_env_guard_count}</p>
+    <p>Graph Ops: {graph_op_count}</p>
+    <p>Graph Nodes: {graph_node_count}</p>
+    <p>Graph Inputs: {graph_input_count}</p>
+    <h2> Custom Ops </h2>
+    <p> Compliant Custom Ops:</p>
+    {{ for op in compliant_custom_ops }}
+    <li> <code> {op} </code> </li>
+    {{ endfor }}
+    <p> Non-Compliant Custom Ops:</p>
+    {{ for op in non_compliant_ops }}
+    <li> <code> {op} </code> </li>
+    {{ endfor }}
+</body>
+</html>
+"#;